diff --git a/.gitignore b/.gitignore index 93aa4834..9c817a77 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ dist-ssr .env* .cache ./logs/ +venv/ # Ignore scratch files - used for testing and development in Data Warehouse scratch* @@ -32,4 +33,4 @@ nohup.out # Service-specific temp files services/*/temp_migration/ services/*/dev-*.log -services/*/.moose/ +services/*/.moose/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 5a16e855..d66a2a14 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -26,5 +26,20 @@ ".*[Cc]lasses.*" ], "tailwindCSS.rootFontSize": 16, - "tailwindCSS.hooverPreview": true + "tailwindCSS.hooverPreview": true, + "sqltools.connections": [ + { + "server": "http://localhost", + "port": 18123, + "database": "default", + "useJWT": false, + "requestTimeout": 30000, + "enableTls": false, + "previewLimit": 50, + "password": "pandapass", + "username": "panda", + "driver": "ClickHouse", + "name": "local" + } + ] } diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..a09cf2f7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,224 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Area Code is a starter repository with multi-modal backend capabilities combining transactional (PostgreSQL), analytical (ClickHouse), and search (Elasticsearch) systems. Built with Turborepo, it includes two main applications: User Facing Analytics (UFA) and Operational Data Warehouse (ODW), plus a lightweight UFA-Lite variant. + +## Package Manager & Node Version + +- **Always use pnpm** (never npm) +- **Required Node version: 20+** (required for Moose framework) +- Package manager: `pnpm@10.14.0` + +## Development Commands + +### UFA (User Facing Analytics) +```bash +# Start all UFA services +pnpm ufa:dev + +# Clean all UFA services and dependencies +pnpm ufa:dev:clean + +# Seed databases with sample data (1M foo records, 100K bar records) +pnpm ufa:dev:seed + +# Seed specific services +pnpm ufa:dev:seed:transactional-supabase-foobar +pnpm ufa:dev:seed:analytical-moose-foobar +pnpm ufa:dev:seed:retrieval-elasticsearch-foobar +``` + +### UFA-Lite (without Elasticsearch) +```bash +# Start UFA-Lite services +pnpm ufa-lite:dev + +# Clean UFA-Lite services +pnpm ufa-lite:dev:clean + +# Seed UFA-Lite databases +pnpm ufa-lite:dev:seed +``` + +### ODW (Operational Data Warehouse) +```bash +# Start ODW services +pnpm odw:dev + +# Clean ODW services +pnpm odw:dev:clean + +# Seed ODW databases +pnpm odw:dev:seed +``` + +### Individual Service Development +```bash +# Frontend only +pnpm --filter web-frontend-foobar dev + +# Transactional API only +pnpm --filter transactional-supabase-foobar dev + +# Analytical API only +pnpm --filter analytical-moose-foobar dev + +# Search API only (UFA only) +pnpm --filter retrieval-elasticsearch-foobar dev +``` + +### Testing and Quality +```bash +# Build all packages +turbo build + +# Lint all packages +turbo lint + +# Type checking (service-specific) +pnpm --filter typecheck +``` + +## Project Structure + +This is a **Turborepo monorepo** with workspaces organized as: + +``` +area-code/ +├── ufa/ # Full-featured UFA +│ ├── apps/ # User-facing applications +│ │ └── web-frontend-foobar/ +│ ├── services/ # Backend services +│ │ ├── transactional-supabase-foobar/ # PostgreSQL + Fastify +│ │ ├── analytical-moose-foobar/ # ClickHouse + Moose +│ │ ├── retrieval-elasticsearch-foobar/ # Elasticsearch +│ │ └── sync-supabase-moose-foobar/ # Data sync workflows +│ └── packages/ # Shared packages +│ ├── models/ # Data models +│ ├── ui/ # UI components +│ ├── eslint-config/ # ESLint config +│ ├── typescript-config/ +│ └── tailwind-config/ +├── ufa-lite/ # Lightweight UFA (no Elasticsearch) +│ ├── apps/ +│ └── services/ +└── odw/ # Operational Data Warehouse + ├── apps/ + ├── services/ + └── packages/ +``` + +## Architecture Components + +### UFA Stack +- **Frontend**: Vite + React 19 + TypeScript + TanStack (Router, Query, Form, Table) + Tailwind CSS +- **Transactional**: PostgreSQL + Fastify + Drizzle ORM + Supabase Realtime +- **Analytical**: ClickHouse + Moose framework (API & Ingest) +- **Search**: Elasticsearch (UFA only, not in UFA-Lite) +- **Sync & Streaming**: Moose Workflows (Temporal) + Moose Stream (Redpanda) + +### Service Ports +**UFA Ports:** +- Frontend: http://localhost:5173 +- Transactional API: http://localhost:8080 +- Analytical Moose: http://localhost:4000 (proxy 4001, management 5001) +- Retrieval API: http://localhost:8081 +- Sync Moose: http://localhost:4100 (management 5101) + +**UFA-Lite Ports:** +- Transactional API: http://localhost:8082 +- Analytical Moose: http://localhost:4410 (proxy 4411, management 5411) +- Sync Moose: http://localhost:4400 (management 5401) + +### Key Technologies +- **Moose**: Framework for analytical APIs, streaming pipelines, and workflows +- **Drizzle ORM**: TypeScript-first ORM for PostgreSQL +- **Supabase**: PostgreSQL with realtime subscriptions +- **ClickHouse**: Analytical database +- **Elasticsearch**: Search engine (UFA only) +- **Temporal**: Workflow orchestration +- **Redpanda**: Event streaming + +## Development Workflow + +1. **Installation**: `pnpm install` (workspace root) +2. **Start Services**: Use appropriate `pnpm :dev` command +3. **Seed Data**: Use `pnpm :dev:seed` for sample data +4. **Individual Development**: Use `--filter` for specific services + +### Moose Service Development +- Moose services support **hot reload** - no manual restart needed +- Test workflows: `moose workflow run ` +- Moose CLI commands: `moose-cli dev`, `moose-cli build`, `moose-cli clean` + +### Package Naming Convention +- Prefix shared packages with `@workspace/` +- Examples: `@workspace/models`, `@workspace/ui`, `@repo/eslint-config` + +## Environment Setup + +### Required for AI Chat Feature (Optional) +Create `.env.local` in `services/transactional-supabase-foobar/`: +```bash +ANTHROPIC_API_KEY=your-api-key-here +``` + +### UFA-Lite Frontend Environment +Create `.env.development.local` in `ufa-lite/apps/web-frontend-foobar/`: +```bash +VITE_ENABLE_SEARCH=false +VITE_TRANSACTIONAL_API_BASE=http://localhost:8082 +VITE_ANALYTICAL_CONSUMPTION_API_BASE=http://localhost:4410 +VITE_SUPABASE_URL=http://localhost:54321 +VITE_SUPABASE_ANON_KEY=dev-anon-key +``` + +## Database Management + +### Transactional Services (PostgreSQL/Supabase) +```bash +# Start database +pnpm --filter transactional-supabase-foobar db:start + +# Stop database +pnpm --filter transactional-supabase-foobar db:stop + +# Run migrations +pnpm --filter transactional-supabase-foobar db:migrate + +# Generate migrations +pnpm --filter transactional-supabase-foobar db:generate +``` + +## Troubleshooting + +### Memory Requirements +- Elasticsearch requires 4GB+ RAM +- Tested on Mac M3/M4 Pro with 18GB+ RAM + +### Reset Environment +```bash +# Clean all services and dependencies +pnpm :dev:clean + +# Restart development +pnpm :dev +``` + +### Common Issues +1. **Node Version**: Ensure Node 20+ for Moose compatibility +2. **Memory**: Insufficient RAM for Elasticsearch +3. **Port Conflicts**: Check that required ports are available +4. **Service Dependencies**: Ensure all services start in correct order + +## Important Notes + +- **Never use npm** - always use pnpm +- **Don't override .env files** - use .env.local for local overrides +- Moose services require Node 20+ +- UFA-Lite omits Elasticsearch for lighter resource usage +- All stacks share containers (Postgres, ClickHouse, Redpanda, Temporal) +- Use `tmux-agent-cmd.sh` wrapper for command execution if available \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/.gitignore b/ufa-lite/services/analytical-moose-foobar-py/.gitignore new file mode 100644 index 00000000..19dcb0d1 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/.gitignore @@ -0,0 +1,57 @@ +# Moose specific +.moose +.sloan + +# Python bytecode and cache +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.mypy_cache +.hypothesis +.coverage + +# Python virtual environments +.Python +env +.venv +venv +ENV +env.bak + +# IDE and editor files +.spyderproject +.ropeproject +.idea +*.ipynb_checkpoints +.cache +.cursor + +# Build and distribution +*.so +*.egg +*.egg-info +dist +build +develop-eggs +downloads +eggs +lib +lib64 +parts +sdist +var +wheels +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Coverage reports +cover +*.cover + +# OS specific +.DS_Store + diff --git a/ufa-lite/services/analytical-moose-foobar-py/.vscode/extensions.json b/ufa-lite/services/analytical-moose-foobar-py/.vscode/extensions.json new file mode 100644 index 00000000..dbc86167 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + "recommendations": [ + "frigus02.vscode-sql-tagged-template-literals-syntax-only", + "mtxr.sqltools", + "ultram4rine.sqltools-clickhouse-driver", + "jeppeandersen.vscode-kafka", + "rangav.vscode-thunder-client" + ] +} diff --git a/ufa-lite/services/analytical-moose-foobar-py/.vscode/settings.json b/ufa-lite/services/analytical-moose-foobar-py/.vscode/settings.json new file mode 100644 index 00000000..48a46372 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/.vscode/settings.json @@ -0,0 +1,17 @@ +{ + "sqltools.connections": [ + { + "server": "localhost", + "port": 18123, + "useHTTPS": false, + "database": "local", + "username": "panda", + "enableTls": false, + "password": "pandapass", + "driver": "ClickHouse", + "name": "moose clickhouse" + } + ], + "python.analysis.extraPaths": [".moose/versions"], + "python.analysis.typeCheckingMode": "basic" +} diff --git a/ufa-lite/services/analytical-moose-foobar-py/README.md b/ufa-lite/services/analytical-moose-foobar-py/README.md new file mode 100644 index 00000000..7f0ec736 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/README.md @@ -0,0 +1,48 @@ +# Template: Python (Empty) + +This is an empty Python-based Moose template that provides a minimal foundation for building data-intensive applications using Python. + +[![PyPI Version](https://img.shields.io/pypi/v/moose-cli?logo=python)](https://pypi.org/project/moose-cli/) +[![Moose Community](https://img.shields.io/badge/slack-moose_community-purple.svg?logo=slack)](https://join.slack.com/t/moose-community/shared_invite/zt-2fjh5n3wz-cnOmM9Xe9DYAgQrNu8xKxg) +[![Docs](https://img.shields.io/badge/quick_start-docs-blue.svg)](https://docs.fiveonefour.com/moose/getting-started/quickstart) +[![MIT license](https://img.shields.io/badge/license-MIT-yellow.svg)](LICENSE) + +## Getting Started + +### Prerequisites + +* [Docker Desktop](https://www.docker.com/products/docker-desktop/) +* [Python](https://www.python.org/downloads/) (version 3.8+) +* [An Anthropic API Key](https://docs.anthropic.com/en/api/getting-started) +* [Cursor](https://www.cursor.com/) or [Claude Desktop](https://claude.ai/download) + +### Installation + +1. Install Moose CLI: `pip install moose-cli` +2. Create project: `moose init python-empty` +3. Install dependencies: `cd && pip install -r requirements.txt` +4. Run Moose: `moose dev` + +You are ready to go! You can start editing the app by modifying primitives in the `app` subdirectory. + +## Learn More + +To learn more about Moose, take a look at the following resources: + +- [Moose Documentation](https://docs.fiveonefour.com/moose) - learn about Moose. +- [Sloan Documentation](https://docs.fiveonefour.com/sloan) - learn about Sloan, the MCP interface for data engineering. + +## Community + +You can join the Moose community [on Slack](https://join.slack.com/t/moose-community/shared_invite/zt-2fjh5n3wz-cnOmM9Xe9DYAgQrNu8xKxg). Check out the [MooseStack repo on GitHub](https://github.com/514-labs/moosestack). + +## Deploy on Boreal + +The easiest way to deploy your MooseStack Applications is to use [Boreal](https://www.fiveonefour.com/boreal) from 514 Labs, the creators of Moose. + +[Sign up](https://www.boreal.cloud/sign-up). + +## License + +This template is MIT licensed. + diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/__init__.py new file mode 100644 index 00000000..4c5284a9 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/__init__.py @@ -0,0 +1,22 @@ +# All APIs for the analytical service +from .foo.consumption import ( + foo_consumption_api, + foo_cube_aggregations_api, + foo_filters_values_api, + foo_score_over_time_api +) +from .bar.consumption import ( + bar_consumption_api, + bar_average_value_api +) + +__all__ = [ + # Foo APIs + "foo_consumption_api", + "foo_cube_aggregations_api", + "foo_filters_values_api", + "foo_score_over_time_api", + # Bar APIs + "bar_consumption_api", + "bar_average_value_api" +] \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/__init__.py new file mode 100644 index 00000000..2d12f3ef --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/__init__.py @@ -0,0 +1 @@ +# Bar APIs module \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/__init__.py new file mode 100644 index 00000000..83f0b3d6 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/__init__.py @@ -0,0 +1,8 @@ +# Bar consumption APIs +from .bar_base_api import bar_consumption_api +from .bar_average_value_api import bar_average_value_api + +__all__ = [ + "bar_consumption_api", + "bar_average_value_api" +] \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/bar_average_value_api.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/bar_average_value_api.py new file mode 100644 index 00000000..c8a00d2a --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/bar_average_value_api.py @@ -0,0 +1,53 @@ +from moose_lib import Api, MooseClient +from typing import Dict, Any +from pydantic import BaseModel +from app.external_models import bar_table +import time + + +class EmptyParams(BaseModel): + """Empty parameters for endpoints with no input parameters""" + pass + + +class GetBarsAverageValueResponse(BaseModel): + averageValue: float + queryTime: int + count: int + + +def bar_average_value_api_handler( + client: MooseClient, + params: EmptyParams +) -> GetBarsAverageValueResponse: + """ + API to get average value of all bars + """ + start_time = time.time() + + query = f""" + SELECT + AVG({bar_table.columns.value}) as averageValue, + COUNT(*) as count + FROM {bar_table.name} + WHERE {bar_table.columns.value} IS NOT NULL + """ + + results = client.query(query, {}) + + query_time = int((time.time() - start_time) * 1000) + + result = results[0] if results else {"averageValue": 0, "count": 0} + + return GetBarsAverageValueResponse( + averageValue=float(result["averageValue"]), + queryTime=query_time, + count=int(result["count"]) + ) + + +# Create the API instance +bar_average_value_api = Api[EmptyParams, GetBarsAverageValueResponse]( + name="bar-average-value", + query_function=bar_average_value_api_handler +) \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/bar_base_api.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/bar_base_api.py new file mode 100644 index 00000000..b52b6055 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/bar/consumption/bar_base_api.py @@ -0,0 +1,74 @@ +from moose_lib import Api, MooseClient +from typing import Optional, List +from pydantic import BaseModel +from app.external_models import bar, bar_table +import time +from datetime import datetime, timezone + +class GetBarsParams(BaseModel): + limit: Optional[int] = 10 + offset: Optional[int] = 0 + sort_by: Optional[str] = "created_at" + sort_order: Optional[str] = "DESC" + +class PaginationInfo(BaseModel): + limit: int + offset: int + total: int + hasMore: bool + + +class GetBarsResponse(BaseModel): + data: List[bar] + pagination: PaginationInfo + queryTime: int + + +def bar_consumption_api_handler(client: MooseClient, params: GetBarsParams) -> GetBarsResponse: + """ + Consumption API for bar data following Moose documentation pattern + """ + # Convert sort_order to uppercase for consistency + upper_sort_order = params.sort_order.upper() + + # Get total count + count_query = f"SELECT count() as total FROM {bar_table.name}" + count_result = client.query(count_query, {}) + total_count = count_result[0]["total"] if count_result else 0 + + start_time = time.time() + + # Build dynamic query including CDC fields + query = f""" + SELECT * + FROM {bar_table.name} + ORDER BY {params.sort_by} {upper_sort_order} + LIMIT {params.limit} + OFFSET {params.offset} + """ + # Run the query + results = client.query(query, {}) + + # Calculate query time + query_time = int((time.time() - start_time) * 1000) # Convert to milliseconds + + # Create pagination metadata + has_more = params.offset + len(results) < total_count + + return GetBarsResponse( + data=results, + pagination=PaginationInfo( + limit=params.limit, + offset=params.offset, + total=total_count, + hasMore=has_more + ), + queryTime=query_time + ) + + +# Create the API instance +bar_consumption_api = Api[GetBarsParams, GetBarsResponse]( + name="bar", + query_function=bar_consumption_api_handler +) \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/__init__.py new file mode 100644 index 00000000..906d034b --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/__init__.py @@ -0,0 +1 @@ +# Foo APIs module \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/__init__.py new file mode 100644 index 00000000..9f9775b5 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/__init__.py @@ -0,0 +1,12 @@ +# Foo consumption APIs +from .foo_base_api import foo_consumption_api +from .foo_cube_aggregations_api import foo_cube_aggregations_api +from .foo_filters_values_api import foo_filters_values_api +from .foo_score_over_time_api import foo_score_over_time_api + +__all__ = [ + "foo_consumption_api", + "foo_cube_aggregations_api", + "foo_filters_values_api", + "foo_score_over_time_api" +] \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_base_api.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_base_api.py new file mode 100644 index 00000000..64d91a2a --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_base_api.py @@ -0,0 +1,72 @@ +from moose_lib import Api, MooseClient +from typing import Optional, List +from pydantic import BaseModel +from app.external_models import foo, foo_table +import time + +class GetFoosParams(BaseModel): + limit: Optional[int] = 10 + offset: Optional[int] = 0 + sort_by: Optional[str] = "created_at" + sort_order: Optional[str] = "DESC" + +class PaginationInfo(BaseModel): + limit: int + offset: int + total: int + hasMore: bool + + +class GetFoosResponse(BaseModel): + data: List[foo] + pagination: PaginationInfo + queryTime: int + + +def foo_consumption_api_handler(client: MooseClient, params: GetFoosParams) -> GetFoosResponse: + """ + Consumption API for foo data following Moose documentation pattern + """ + # Convert sort_order to uppercase for consistency + upper_sort_order = params.sort_order.upper() + + # Get total count + count_query = f"SELECT count() as total FROM {foo_table.name}" + count_result = client.query(count_query, {}) + total_count = count_result[0]["total"] if count_result else 0 + + start_time = time.time() + + # Build dynamic query including CDC fields + query = f""" + SELECT * + FROM {foo_table.name} + ORDER BY {params.sort_by} {upper_sort_order} + LIMIT {params.limit} + OFFSET {params.offset} + """ + # Run the query + results = client.query(query, {}) + + # Calculate query time + query_time = int((time.time() - start_time) * 1000) # Convert to milliseconds + + # Create pagination metadata + has_more = params.offset + len(results) < total_count + + return GetFoosResponse( + data=results, + pagination=PaginationInfo( + limit=params.limit, + offset=params.offset, + total=total_count, + hasMore=has_more + ), + queryTime=query_time + ) + +# Create the API instance +foo_consumption_api = Api[GetFoosParams, GetFoosResponse]( + name="foo", + query_function=foo_consumption_api_handler +) \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_cube_aggregations_api.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_cube_aggregations_api.py new file mode 100644 index 00000000..8156fb8f --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_cube_aggregations_api.py @@ -0,0 +1,142 @@ +from moose_lib import Api, MooseClient +from typing import Optional, List +from pydantic import BaseModel +from app.external_models import foo_table +import time +from datetime import datetime, timedelta + + +class GetFooCubeAggregationsParams(BaseModel): + months: Optional[int] = 6 + status: Optional[str] = None + tag: Optional[str] = None + priority: Optional[int] = None + limit: Optional[int] = 20 + offset: Optional[int] = 0 + sort_by: Optional[str] = None + sort_order: Optional[str] = "ASC" + + +class FooCubeAggregationRow(BaseModel): + month: Optional[str] = None + status: Optional[str] = None + tag: Optional[str] = None + priority: Optional[int] = None + n: int + avgScore: float + p50: float + p90: float + + +class GetFooCubeAggregationsResponse(BaseModel): + data: List[FooCubeAggregationRow] + queryTime: int + pagination: dict + + +def foo_cube_aggregations_api_handler( + client: MooseClient, + params: GetFooCubeAggregationsParams +) -> GetFooCubeAggregationsResponse: + """ + Cube aggregations API for foo data with monthly grouping and statistical analysis + """ + start_time = time.time() + + # Calculate date range + end_date = datetime.now() + start_date = end_date - timedelta(days=max(1, min(params.months, 36)) * 30) + + start_date_str = start_date.strftime("%Y-%m-%d") + end_date_str = end_date.strftime("%Y-%m-%d") + + # Build optional WHERE fragments + where_clauses = [ + f"toDate({foo_table.columns.created_at}) >= toDate('{start_date_str}')", + f"toDate({foo_table.columns.created_at}) <= toDate('{end_date_str}')", + f"{foo_table.columns.score} IS NOT NULL" + ] + + if params.status: + where_clauses.append(f"{foo_table.columns.status} = '{params.status}'") + + if params.priority is not None: + where_clauses.append(f"{foo_table.columns.priority} = {params.priority}") + + limited = max(1, min(params.limit, 200)) + paged_offset = max(0, params.offset) + + # Map sort column safely + sort_column = { + "month": "month", + "status": f"{foo_table.columns.status}", + "tag": "tag", + "priority": f"{foo_table.columns.priority}", + "n": "n", + "avgScore": "avgScore", + "avg_score": "avgScore", + "p50": "p50", + "p90": "p90" + }.get(params.sort_by, "month, status, tag, priority") + + sort_dir = "DESC" if params.sort_order.upper() == "DESC" else "ASC" + + # Build the main query + where_clause = " AND ".join(where_clauses) + having_clause = f"HAVING tag = '{params.tag}'" if params.tag else "" + + query = f""" + SELECT + formatDateTime(toStartOfMonth({foo_table.columns.created_at}), '%Y-%m-01') AS month, + {foo_table.columns.status}, + arrayJoin({foo_table.columns.tags}) AS tag, + {foo_table.columns.priority}, + count() AS n, + avg({foo_table.columns.score}) AS avgScore, + quantileTDigest(0.5)(toFloat64({foo_table.columns.score})) AS p50, + quantileTDigest(0.9)(toFloat64({foo_table.columns.score})) AS p90, + COUNT() OVER() AS total + FROM {foo_table.name} + WHERE {where_clause} + GROUP BY month, {foo_table.columns.status}, tag, {foo_table.columns.priority} + {having_clause} + ORDER BY {sort_column} {sort_dir} + LIMIT {limited} OFFSET {paged_offset} + """ + + results = client.query(query, {}) + + query_time = int((time.time() - start_time) * 1000) + total = results[0]["total"] if results else 0 + + # Convert results to response format + data = [ + FooCubeAggregationRow( + month=row["month"], + status=row["status"], + tag=row["tag"], + priority=int(row["priority"]) if row["priority"] is not None else None, + n=int(row["n"]), + avgScore=float(row["avgScore"]), + p50=float(row["p50"]), + p90=float(row["p90"]) + ) + for row in results + ] + + return GetFooCubeAggregationsResponse( + data=data, + queryTime=query_time, + pagination={ + "limit": limited, + "offset": paged_offset, + "total": total + } + ) + + +# Create the API instance +foo_cube_aggregations_api = Api[GetFooCubeAggregationsParams, GetFooCubeAggregationsResponse]( + name="foo-cube-aggregations", + query_function=foo_cube_aggregations_api_handler +) \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_filters_values_api.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_filters_values_api.py new file mode 100644 index 00000000..d36fc916 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_filters_values_api.py @@ -0,0 +1,83 @@ +from moose_lib import Api, MooseClient +from typing import Optional, List +from pydantic import BaseModel +from app.external_models import foo_table +from datetime import datetime, timedelta + + +class GetFooFiltersValuesParams(BaseModel): + months: Optional[int] = 6 + + +class GetFooFiltersValuesResponse(BaseModel): + status: List[str] + tags: List[str] + priorities: List[int] + + +def foo_filters_values_api_handler( + client: MooseClient, + params: GetFooFiltersValuesParams +) -> GetFooFiltersValuesResponse: + """ + API to get distinct filter values for foo data + """ + # Calculate date range + end_date = datetime.now() + start_date = end_date - timedelta(days=max(1, min(params.months, 36)) * 30) + + start_date_str = start_date.strftime("%Y-%m-%d") + end_date_str = end_date.strftime("%Y-%m-%d") + + # Build queries for distinct values + date_filter = f""" + toDate({foo_table.columns.created_at}) >= toDate('{start_date_str}') + AND toDate({foo_table.columns.created_at}) <= toDate('{end_date_str}') + """ + + status_query = f""" + SELECT DISTINCT {foo_table.columns.status} + FROM {foo_table.name} + WHERE {date_filter} + AND {foo_table.columns.status} IS NOT NULL + ORDER BY {foo_table.columns.status} + """ + + tags_query = f""" + SELECT DISTINCT arrayJoin({foo_table.columns.tags}) AS tag + FROM {foo_table.name} + WHERE {date_filter} + AND {foo_table.columns.tags} IS NOT NULL + ORDER BY tag + """ + + priorities_query = f""" + SELECT DISTINCT {foo_table.columns.priority} + FROM {foo_table.name} + WHERE {date_filter} + AND {foo_table.columns.priority} IS NOT NULL + ORDER BY {foo_table.columns.priority} + """ + + # Execute queries + status_results = client.query(status_query, {}) + tags_results = client.query(tags_query, {}) + priorities_results = client.query(priorities_query, {}) + + # Extract values from results + possible_statuses = [row["status"] for row in status_results] + possible_tags = [row["tag"] for row in tags_results] + possible_priorities = [int(row["priority"]) for row in priorities_results] + + return GetFooFiltersValuesResponse( + status=possible_statuses, + tags=possible_tags, + priorities=possible_priorities + ) + + +# Create the API instance +foo_filters_values_api = Api[GetFooFiltersValuesParams, GetFooFiltersValuesResponse]( + name="foo-filters-values", + query_function=foo_filters_values_api_handler +) \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_score_over_time_api.py b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_score_over_time_api.py new file mode 100644 index 00000000..9a647a33 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/apis/foo/consumption/foo_score_over_time_api.py @@ -0,0 +1,79 @@ +from moose_lib import Api, MooseClient +from typing import Optional, List +from pydantic import BaseModel +from app.external_models import foo_table +import time +from datetime import datetime, timedelta + + +class FoosScoreOverTimeDataPoint(BaseModel): + date: str + averageScore: float + totalCount: int + + +class GetFoosScoreOverTimeParams(BaseModel): + days: Optional[int] = 90 + + +class GetFoosScoreOverTimeResponse(BaseModel): + data: List[FoosScoreOverTimeDataPoint] + queryTime: int + + +def foo_score_over_time_api_handler( + client: MooseClient, + params: GetFoosScoreOverTimeParams +) -> GetFoosScoreOverTimeResponse: + """ + Score over time consumption API for foo data + """ + # Calculate date range + start_date = datetime.now() - timedelta(days=params.days) + end_date = datetime.now() + + # Format dates for SQL + start_date_str = start_date.strftime("%Y-%m-%d") + end_date_str = end_date.strftime("%Y-%m-%d") + + # Query to get daily score aggregations + query = f""" + SELECT + formatDateTime(toDate({foo_table.columns.created_at}), '%Y-%m-%d') as date, + AVG({foo_table.columns.score}) as averageScore, + COUNT(*) as totalCount + FROM {foo_table.name} + WHERE toDate({foo_table.columns.created_at}) >= toDate('{start_date_str}') + AND toDate({foo_table.columns.created_at}) <= toDate('{end_date_str}') + AND {foo_table.columns.score} IS NOT NULL + GROUP BY toDate({foo_table.columns.created_at}) + ORDER BY toDate({foo_table.columns.created_at}) ASC + """ + + start_time = time.time() + + results = client.query(query, {}) + + query_time = int((time.time() - start_time) * 1000) + + # Convert results to response format + data = [ + FoosScoreOverTimeDataPoint( + date=str(row["date"]), # Date is already formatted as YYYY-MM-DD string + averageScore=round(float(row["averageScore"]), 2), # Round to 2 decimal places + totalCount=int(row["totalCount"]) + ) + for row in results + ] + + return GetFoosScoreOverTimeResponse( + data=data, + queryTime=query_time + ) + + +# Create the API instance +foo_score_over_time_api = Api[GetFoosScoreOverTimeParams, GetFoosScoreOverTimeResponse]( + name="foo-score-over-time", + query_function=foo_score_over_time_api_handler +) \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/external_models.py b/ufa-lite/services/analytical-moose-foobar-py/app/external_models.py new file mode 100644 index 00000000..0c92fe8a --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/external_models.py @@ -0,0 +1,91 @@ +# AUTO-GENERATED FILE. DO NOT EDIT. +# This file will be replaced when you run `moose db pull`. + +from pydantic import BaseModel, Field +from typing import Optional, Any, Annotated +import datetime +import ipaddress +from uuid import UUID +from enum import IntEnum, Enum +from moose_lib import Key, IngestPipeline, IngestPipelineConfig, OlapTable, OlapConfig, clickhouse_datetime64, clickhouse_decimal, ClickhouseSize, StringToEnumMixin +from moose_lib import clickhouse_default, LifeCycle +from moose_lib.blocks import MergeTreeEngine, ReplacingMergeTreeEngine, AggregatingMergeTreeEngine, SummingMergeTreeEngine, S3QueueEngine + +class _peerdb_raw_mirror_a4be3c5e__1df3__45e4__805b__cb363b330b4e(BaseModel): + UNDERSCORE_PREFIXED_peerdb_uid: UUID = Field(alias="_peerdb_uid") + UNDERSCORE_PREFIXED_peerdb_timestamp: Annotated[int, "int64"] = Field(alias="_peerdb_timestamp") + UNDERSCORE_PREFIXED_peerdb_destination_table_name: str = Field(alias="_peerdb_destination_table_name") + UNDERSCORE_PREFIXED_peerdb_data: str = Field(alias="_peerdb_data") + UNDERSCORE_PREFIXED_peerdb_record_type: Annotated[int, "int32"] = Field(alias="_peerdb_record_type") + UNDERSCORE_PREFIXED_peerdb_match_data: str = Field(alias="_peerdb_match_data") + UNDERSCORE_PREFIXED_peerdb_batch_id: Annotated[int, "int64"] = Field(alias="_peerdb_batch_id") + UNDERSCORE_PREFIXED_peerdb_unchanged_toast_columns: str = Field(alias="_peerdb_unchanged_toast_columns") + +class bar(BaseModel): + id: Key[UUID] + foo_id: UUID + value: Annotated[int, "int32"] + label: Optional[str] = None + notes: Optional[str] = None + is_enabled: bool + created_at: clickhouse_datetime64(6) + updated_at: clickhouse_datetime64(6) + UNDERSCORE_PREFIXED_peerdb_synced_at: Annotated[clickhouse_datetime64(9), clickhouse_default("now64()")] = Field(alias="_peerdb_synced_at") + UNDERSCORE_PREFIXED_peerdb_is_deleted: Annotated[int, "int8"] = Field(alias="_peerdb_is_deleted") + UNDERSCORE_PREFIXED_peerdb_version: Annotated[int, "int64"] = Field(alias="_peerdb_version") + +class foo(BaseModel): + id: Key[UUID] + name: str + description: Optional[str] = None + status: str + priority: Annotated[int, "int32"] + is_active: bool + metadata: Optional[str] = None + tags: list[str] + score: Optional[clickhouse_decimal(10, 2)] = None + large_text: Optional[str] = None + created_at: clickhouse_datetime64(6) + updated_at: clickhouse_datetime64(6) + UNDERSCORE_PREFIXED_peerdb_synced_at: Annotated[clickhouse_datetime64(9), clickhouse_default("now64()")] = Field(alias="_peerdb_synced_at") + UNDERSCORE_PREFIXED_peerdb_is_deleted: Annotated[int, "int8"] = Field(alias="_peerdb_is_deleted") + UNDERSCORE_PREFIXED_peerdb_version: Annotated[int, "int64"] = Field(alias="_peerdb_version") + +class users(BaseModel): + id: Key[UUID] + role: str + created_at: clickhouse_datetime64(6) + updated_at: clickhouse_datetime64(6) + UNDERSCORE_PREFIXED_peerdb_synced_at: Annotated[clickhouse_datetime64(9), clickhouse_default("now64()")] = Field(alias="_peerdb_synced_at") + UNDERSCORE_PREFIXED_peerdb_is_deleted: Annotated[int, "int8"] = Field(alias="_peerdb_is_deleted") + UNDERSCORE_PREFIXED_peerdb_version: Annotated[int, "int64"] = Field(alias="_peerdb_version") + +peerdb_raw_mirror_a_4_be_3_c_5_e_1_df_3_45_e_4_805_b_cb_363_b_330_b_4_e_table = OlapTable[_peerdb_raw_mirror_a4be3c5e__1df3__45e4__805b__cb363b330b4e]("_peerdb_raw_mirror_a4be3c5e__1df3__45e4__805b__cb363b330b4e", OlapConfig( + order_by_fields=["_peerdb_batch_id", "_peerdb_destination_table_name"], + life_cycle=LifeCycle.EXTERNALLY_MANAGED, + engine=MergeTreeEngine(), + settings={"index_granularity": "8192"}, +)) + +bar_table = OlapTable[bar]("bar", OlapConfig( + order_by_fields=["id"], + life_cycle=LifeCycle.EXTERNALLY_MANAGED, + engine=ReplacingMergeTreeEngine(ver="_peerdb_version"), + settings={"index_granularity": "8192"}, +)) + +foo_table = OlapTable[foo]("foo", OlapConfig( + order_by_fields=["id"], + life_cycle=LifeCycle.EXTERNALLY_MANAGED, + engine=ReplacingMergeTreeEngine(ver="_peerdb_version"), + settings={"index_granularity": "8192"}, +)) + +users_table = OlapTable[users]("users", OlapConfig( + order_by_fields=["id"], + life_cycle=LifeCycle.EXTERNALLY_MANAGED, + engine=ReplacingMergeTreeEngine(ver="_peerdb_version"), + settings={"index_granularity": "8192"}, +)) + + diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/ingest/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/ingest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/main.py b/ufa-lite/services/analytical-moose-foobar-py/app/main.py new file mode 100644 index 00000000..54eabe85 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/app/main.py @@ -0,0 +1,66 @@ +# Welcome to your new Moose analytical backend! 🦌 + +# Getting Started Guide: + +# 1. Data Modeling +# First, plan your data structure and create your data models +# → See: docs.fiveonefour.com/moose/building/data-modeling +# Learn about type definitions and data validation + +# 2. Set Up Ingestion +# Create ingestion pipelines to receive your data via REST APIs +# → See: docs.fiveonefour.com/moose/building/ingestion +# Learn about IngestPipeline, data formats, and validation + +# 3. Create Workflows +# Build data processing pipelines to transform and analyze your data +# → See: docs.fiveonefour.com/moose/building/workflows +# Learn about task scheduling and data processing + +# 4. Configure Consumption APIs +# Set up queries and real-time analytics for your data +# → See: docs.fiveonefour.com/moose/building/consumption-apis + +# Need help? Check out the quickstart guide: +# → docs.fiveonefour.com/moose/getting-started/quickstart + +# Import external models for their side effects: registers additional data models required for pipeline and API setup +from .external_models import * + +# Import and register all consumption APIs +from .apis.bar.consumption.bar_average_value_api import bar_average_value_api +from .apis.bar.consumption.bar_base_api import bar_consumption_api +from .apis.foo.consumption.foo_base_api import foo_consumption_api +from .apis.foo.consumption.foo_score_over_time_api import foo_score_over_time_api +from .apis.foo.consumption.foo_cube_aggregations_api import foo_cube_aggregations_api +from .apis.foo.consumption.foo_filters_values_api import foo_filters_values_api + + +from pydantic import BaseModel, Field +from typing import Optional, Any, Annotated +import datetime +import ipaddress +from uuid import UUID +from enum import IntEnum, Enum +from moose_lib import Key, IngestPipeline, IngestPipelineConfig, OlapTable, OlapConfig, clickhouse_datetime64, clickhouse_decimal, ClickhouseSize, StringToEnumMixin +from moose_lib import clickhouse_default, LifeCycle +from moose_lib.blocks import MergeTreeEngine, ReplacingMergeTreeEngine, AggregatingMergeTreeEngine, SummingMergeTreeEngine, S3QueueEngine + +class dish(BaseModel): + id: Annotated[int, "uint32"] + name: str + description: str + menus_appeared: Annotated[int, "uint32"] + times_appeared: Annotated[int, "int32"] + first_appeared: Annotated[int, "uint16"] + last_appeared: Annotated[int, "uint16"] + lowest_price: clickhouse_decimal(18, 3) + highest_price: clickhouse_decimal(18, 3) + +dish_table = OlapTable[dish]("dish", OlapConfig( + order_by_fields=["id"], + engine=MergeTreeEngine(), + settings={"index_granularity": "8192"}, +)) + + diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/scripts/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ufa-lite/services/analytical-moose-foobar-py/app/views/__init__.py b/ufa-lite/services/analytical-moose-foobar-py/app/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ufa-lite/services/analytical-moose-foobar-py/moose.config.toml b/ufa-lite/services/analytical-moose-foobar-py/moose.config.toml new file mode 100644 index 00000000..5039f753 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/moose.config.toml @@ -0,0 +1,69 @@ +language = "Python" + +[redpanda_config] +broker = "localhost:19092" +message_timeout_ms = 1000 +retention_ms = 30000 +replication_factor = 1 + +[clickhouse_config] +db_name = "local" +user = "panda" +password = "pandapass" +use_ssl = false +host = "localhost" +host_port = 18123 +native_port = 9000 + +[http_server_config] +host = "localhost" +port = 4410 +management_port = 5411 +proxy_port = 4411 +max_request_body_size = 10485760 + +[redis_config] +url = "redis://127.0.0.1:6379" +key_prefix = "MS" +last_key_prefix = "MS" +port = 6379 +tls = false +hostname = "127.0.0.1" + +[git_config] +main_branch_name = "main" + +[temporal_config] +db_user = "temporal" +db_password = "temporal" +db_port = 5432 +namespace = "default" +temporal_host = "localhost" +temporal_port = 7233 +temporal_version = "1.22.3" +temporal_region = "us-west1" +admin_tools_version = "1.22.3" +ui_version = "2.21.3" +ui_port = 8080 +ui_cors_origins = "http://localhost:3000" +config_path = "config/dynamicconfig/development-sql.yaml" +postgresql_version = "13" +client_cert = "" +client_key = "" +ca_cert = "" +api_key = "" + +[supported_old_versions] + +[authentication] +admin_api_key = "199d13d8201f7418054de884f3a955c928122ab3" + +[features] +streaming_engine = false +workflows = false +data_model_v2 = true +olap = true +ddl_plan = false + +[typescript_config] +package_manager = "npm" diff --git a/ufa-lite/services/analytical-moose-foobar-py/requirements.txt b/ufa-lite/services/analytical-moose-foobar-py/requirements.txt new file mode 100644 index 00000000..6c427d42 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/requirements.txt @@ -0,0 +1,5 @@ +kafka-python-ng==2.2.2 +clickhouse-connect==0.7.16 +requests==2.32.4 +moose-cli +moose-lib \ No newline at end of file diff --git a/ufa-lite/services/analytical-moose-foobar-py/setup.py b/ufa-lite/services/analytical-moose-foobar-py/setup.py new file mode 100644 index 00000000..b6873d26 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/setup.py @@ -0,0 +1,12 @@ + +from setuptools import setup + +with open('requirements.txt') as f: + requirements = [line.strip() for line in f if line.strip() and not line.startswith('#')] + +setup( + name='analytical-moose-foobar-py', + version='0.0', + install_requires=requirements, + python_requires='>=3.12', +) diff --git a/ufa-lite/services/analytical-moose-foobar-py/template.config.toml b/ufa-lite/services/analytical-moose-foobar-py/template.config.toml new file mode 100644 index 00000000..266c0c29 --- /dev/null +++ b/ufa-lite/services/analytical-moose-foobar-py/template.config.toml @@ -0,0 +1,26 @@ +language = "python" # Must be typescript or python +description = "Empty python project." +post_install_print = """ +Deploy on Boreal + +The easiest way to deploy your MooseStack Applications is to use Boreal +from the creators of MooseStack. + +https://boreal.cloud + +--------------------------------------------------------- + +📂 Go to your project directory: + $ cd {project_dir} + +🥄 Create a virtual environment (optional, recommended): + $ python3 -m venv .venv + $ source .venv/bin/activate + +📦 Install Dependencies: + $ pip install -r ./requirements.txt + +🛠️ Start dev server: + $ moose dev +""" +default_sloan_telemetry="standard"