From f119f60f1b75a43be2537aef4c87cba00890ab9b Mon Sep 17 00:00:00 2001 From: Gal Elmalah Date: Wed, 4 Feb 2026 09:33:49 +0200 Subject: [PATCH 1/3] docs: add JSON Schema and documentation for evals Add IDE support and comprehensive documentation for the eval test framework: - Create eval.schema.json with definitions for all 9 check types - Update README.md with detailed check type reference and examples - Add $schema reference to all 7 fixture eval.json files The schema enables autocomplete and validation in VS Code and other editors when editing eval.json files. Co-Authored-By: Claude Opus 4.5 --- evals/README.md | 449 ++++++++++++++++++ evals/eval.schema.json | 233 +++++++++ evals/fixtures/agent-chat-component/eval.json | 43 ++ evals/fixtures/agent-with-function/eval.json | 52 ++ evals/fixtures/create-support-agent/eval.json | 48 ++ .../existing-app-add-feature/eval.json | 53 +++ evals/fixtures/nextjs-todo/eval.json | 52 ++ evals/fixtures/react-task-manager/eval.json | 38 ++ evals/fixtures/skill-check/eval.json | 19 + 9 files changed, 987 insertions(+) create mode 100644 evals/README.md create mode 100644 evals/eval.schema.json create mode 100644 evals/fixtures/agent-chat-component/eval.json create mode 100644 evals/fixtures/agent-with-function/eval.json create mode 100644 evals/fixtures/create-support-agent/eval.json create mode 100644 evals/fixtures/existing-app-add-feature/eval.json create mode 100644 evals/fixtures/nextjs-todo/eval.json create mode 100644 evals/fixtures/react-task-manager/eval.json create mode 100644 evals/fixtures/skill-check/eval.json diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..84ab05e --- /dev/null +++ b/evals/README.md @@ -0,0 +1,449 @@ +# Skills Eval System + +An evaluation system for testing skill activation and result correctness against Claude Code (or other coding agents). + +## Quick Start + +```bash +# Install dependencies +npm install + +# Run all evals (automatically syncs skills from parent directory) +npm run eval + +# Run with a named run +npm run eval -- --name "baseline" + +# Filter to specific fixtures +npm run eval -- --filter nextjs --verbose + +# Run without re-syncing skills (faster for repeated runs) +npm run eval:only -- --name "quick-test" +``` + +### Setup + +The `npm run eval` command automatically runs `npm run setup` first, which copies skills from the parent `../skills/` directory into each fixture. This ensures fixtures always test against the latest skill versions. + +To manually sync skills: +```bash +npm run setup +``` + +## Directory Structure + +``` +evals/ +├── src/ # Source code +│ ├── index.ts # CLI entry point +│ ├── runner.ts # Eval runner orchestration +│ ├── agents/ # Coding agent implementations +│ │ ├── interface.ts # CodingAgent interface +│ │ └── claude-code.ts # Claude Code CLI implementation +│ ├── checks/ # Check implementations +│ │ ├── interface.ts # Check registry +│ │ ├── entity-config.ts # Entity schema validation +│ │ ├── function-def.ts # Function definition validation +│ │ ├── agent-config.ts # Agent config validation +│ │ ├── contains.ts # String containment check +│ │ ├── file-exists.ts # File existence check +│ │ ├── file-content.ts # File content pattern matching +│ │ ├── command-passes.ts # Command execution check +│ │ ├── valid-json.ts # JSON validity check +│ │ └── json-schema.ts # Generic JSON schema validation +│ ├── reporters/ +│ │ └── markdown.ts # Markdown report generator +│ └── types.ts # Shared types +├── fixtures/ # Test fixtures (realistic projects) +│ ├── nextjs-todo/ # Next.js app - add todo entities +│ ├── react-task-manager/ # React app - implement SDK usage +│ └── existing-app-add-feature/ # Add backend function +├── results/ # Generated reports +├── eval.schema.json # JSON Schema for eval.json files +├── package.json +└── tsconfig.json +``` + +## Fixtures + +Each fixture is a **complete, isolated project** that the agent runs against: + +``` +fixture-name/ +├── eval.json # Eval config: prompt, expected outcomes, checks +├── AGENTS.md # Project context for the agent +└── project/ # The actual project directory (cwd for agent) + ├── .claude/ + │ └── settings.json # Claude settings (skills paths) + ├── skills/ # Skills (copied by setup script) + │ ├── base44-cli/ + │ └── base44-sdk/ + ├── base44/ + │ ├── config.jsonc + │ └── entities/ + ├── src/ + ├── package.json + └── ... +``` + +**Important:** Skills are copied into `project/skills/` by the setup script. The `.claude/settings.json` file inside `project/` references these skills. + +### Current Fixtures + +| Fixture | Description | Tests | +|---------|-------------|-------| +| `nextjs-todo` | Next.js app needing todo entities | Entity creation, file structure, CLI commands | +| `react-task-manager` | React app with existing Task entity | SDK usage, API calls, React hooks | +| `existing-app-add-feature` | App needing a backend function | Function creation, Deno patterns, integrations | + +## Creating a New Test + +1. **Create a fixture directory**: + ```bash + mkdir -p fixtures/my-new-test/project + ``` + +2. **Create `eval.json`** with the schema reference for IDE autocomplete: + ```json + { + "$schema": "../../eval.schema.json", + "name": "my-new-test", + "description": "Test that validates specific behavior", + "prompt": "Your prompt to Claude here", + "expectedSkills": ["base44-sdk"], + "checks": [ + { + "type": "contains", + "description": "Output mentions the feature", + "value": "feature" + } + ] + } + ``` + +3. **Add project files**: Create `project/.claude/settings.json` and any starter files needed. + +4. **Add `AGENTS.md`** with project context for the agent. + +5. **Run setup and test**: + ```bash + npm run setup + npm run eval -- --filter my-new-test --verbose + ``` + +## eval.json Format + +```json +{ + "$schema": "../../eval.schema.json", + "name": "nextjs-todo-entities", + "description": "Test creating Todo and Category entities", + "prompt": "Add entities for a todo app...", + "expectedSkills": ["base44-cli", "base44-sdk"], + "checks": [ + { + "type": "file-exists", + "description": "Todo entity file created", + "filePath": "base44/entities/todo.jsonc" + }, + { + "type": "file-content", + "description": "Todo has title property", + "filePath": "base44/entities/todo.jsonc", + "pattern": "\"title\"\\s*:\\s*\\{" + }, + { + "type": "valid-json", + "description": "Todo entity is valid JSON", + "filePath": "base44/entities/todo.jsonc" + }, + { + "type": "contains", + "description": "mentions entities push", + "value": "entities push" + } + ] +} +``` + +The `$schema` reference enables IDE features like autocomplete, validation, and inline documentation in VS Code and other editors. + +## Check Types Reference + +| Type | Description | Required Fields | Optional Fields | +|------|-------------|-----------------|-----------------| +| `contains` | Search for text in agent output | `value` | | +| `file-exists` | Verify a file was created | `filePath` | | +| `file-content` | Match content in a file | `filePath` | `pattern`, `value` | +| `valid-json` | Validate JSON/JSONC syntax | `filePath` | | +| `command-passes` | Run a shell command | `command` | | +| `json-schema` | Validate output against schema | `schema` | `filePath` | +| `entity-config` | Validate Base44 entity config | | `filePath`, `target`, `expectedValid` | +| `agent-config` | Validate Base44 agent config | | `filePath`, `target`, `expectedValid` | +| `function-def` | Validate Base44 function config | | `filePath`, `target`, `expectedValid` | + +### Check Examples + +#### `contains` - Search agent output + +Checks if the agent's text output contains a substring (case-insensitive). + +```json +{ + "type": "contains", + "description": "Output mentions entities", + "value": "entities" +} +``` + +#### `file-exists` - Verify file creation + +Checks if a file exists at the specified path (relative to project directory). + +```json +{ + "type": "file-exists", + "description": "Component file created", + "filePath": "src/components/TaskList.tsx" +} +``` + +#### `file-content` - Match file content with regex + +Checks if file content matches a regular expression pattern (case-insensitive). + +```json +{ + "type": "file-content", + "description": "Uses React hooks", + "filePath": "src/components/TaskList.tsx", + "pattern": "useState|useEffect" +} +``` + +#### `file-content` - Match file content with exact value + +Checks if file contains an exact substring. + +```json +{ + "type": "file-content", + "description": "Imports base44", + "filePath": "src/components/TaskList.tsx", + "value": "import base44" +} +``` + +#### `valid-json` - Validate JSON syntax + +Verifies the file contains valid JSON/JSONC (supports `//` and `/* */` comments). + +```json +{ + "type": "valid-json", + "description": "Config is valid JSON", + "filePath": "base44/entities/todo.jsonc" +} +``` + +#### `command-passes` - Run shell command + +Runs a shell command and passes if exit code is 0. Commands have a 2-minute timeout. + +```json +{ + "type": "command-passes", + "description": "TypeScript compiles", + "command": "npx tsc --noEmit" +} +``` + +#### `json-schema` - Validate against JSON Schema + +Validates JSON in agent output (or a file) against a JSON Schema definition. + +```json +{ + "type": "json-schema", + "description": "Output matches expected structure", + "schema": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" } + }, + "required": ["name"] + } +} +``` + +#### `entity-config` - Validate Base44 entity schema + +Validates that a file contains a valid Base44 entity configuration with `name`, `properties`, etc. + +```json +{ + "type": "entity-config", + "description": "Valid entity schema", + "filePath": "base44/entities/todo.jsonc", + "target": "file" +} +``` + +When `target` is omitted or set to `"output"`, extracts entity JSON from the agent's output. + +#### `agent-config` - Validate Base44 agent config + +Validates that a file contains a valid Base44 agent configuration with `name`, `instructions`, `tool_configs`, etc. + +```json +{ + "type": "agent-config", + "description": "Valid agent configuration", + "filePath": "base44/agents/support_agent.jsonc", + "target": "file" +} +``` + +#### `function-def` - Validate Base44 function config + +Validates that a file contains a valid Base44 function definition with `name`, `entrypoint`, etc. + +```json +{ + "type": "function-def", + "description": "Valid function definition", + "filePath": "base44/functions/my-func/function.jsonc", + "target": "file" +} +``` + +## CLI Options + +``` +-n, --name Name for this eval run (default: "default") +-a, --agent Agent to use (default: "claude-code") +-f, --fixtures Fixtures directory (default: "fixtures") +-o, --output Output directory for reports (default: "results") +-v, --verbose Verbose output +--filter Filter fixtures by name pattern +``` + +## Reports + +Reports are generated in `results/` with timestamped filenames: + +``` +results/ +├── run-2026-02-02-143052-default.md +├── run-2026-02-02-150312-with-new-skill.md +└── latest.md → (symlink to most recent) +``` + +Example report format: + +```markdown +# Eval Run: baseline +**Date**: 2026-02-02 15:03:12 +**Agent**: claude-code +**Fixtures**: 3 + +## Summary +| Status | Count | +|--------|-------| +| ✅ Passed | 2 | +| ❌ Failed | 1 | + +## nextjs-todo Suite + +### ✅ nextjs-todo-entities +**Fixture**: `fixtures/nextjs-todo/` +**Prompt**: Add entities for a todo app... + +- **Expected Skills**: base44-cli, base44-sdk ✓ +- **Skills Invoked**: base44-cli, base44-sdk +- **Checks**: + | Check | Status | Details | + |-------|--------|---------| + | Todo entity file created | ✅ | File exists | + | Todo has title property | ✅ | Pattern found | +``` + +## Adding Custom Checks + +To add a new check type: + +1. **Create the check class** in `src/checks/`: + ```typescript + // src/checks/my-check.ts + import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + + export class MyCheck implements Check { + type = 'my-check'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { agentResponse, projectDir } = context; + + // Your validation logic here + const passed = /* ... */; + + return { + name: this.config.description, + passed, + details: passed ? 'Check passed' : 'Check failed', + }; + } + } + ``` + +2. **Register it** in `src/checks/interface.ts`: + ```typescript + import { MyCheck } from './my-check.js'; + + // In createCheck function: + case 'my-check': + return new MyCheck(config); + ``` + +3. **Update the schema** in `eval.schema.json`: + - Add your type to the `type` enum + - Add a conditional schema for your check's fields + +4. **Update this README** with documentation for your new check type. + +### EvalContext + +Check implementations receive an `EvalContext` with: + +```typescript +interface EvalContext { + agentResponse: { + output: string; // Claude's text response + skillsInvoked: string[]; // Skills that were invoked + metadata?: Record; + }; + fixtureDir: string; // Original fixture directory + projectDir: string; // Temp directory where agent ran + checkConfig: CheckConfig; +} +``` + +## Skill Detection + +The eval system detects skill activation by pattern matching in the agent's output: + +**base44-cli patterns:** +- `npx base44 entities push`, `functions deploy`, etc. +- File paths like `base44/entities/*.jsonc` +- `Deno.serve`, `createClientFromRequest` + +**base44-sdk patterns:** +- `@base44/sdk` imports +- `entities.Task.list()`, `entities.Task.create()` +- `integrations.Core.SendEmail` +- `asServiceRole` diff --git a/evals/eval.schema.json b/evals/eval.schema.json new file mode 100644 index 0000000..60f7a21 --- /dev/null +++ b/evals/eval.schema.json @@ -0,0 +1,233 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Eval Configuration", + "description": "Configuration for a skills eval test case", + "type": "object", + "required": ["name", "description", "prompt"], + "properties": { + "$schema": { + "type": "string", + "description": "JSON Schema reference for IDE support" + }, + "name": { + "type": "string", + "description": "Unique identifier for the test case" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the test validates" + }, + "prompt": { + "type": "string", + "description": "The prompt sent to Claude for this test" + }, + "expectedSkills": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of skill names that should be detected/invoked during the test" + }, + "checks": { + "type": "array", + "items": { + "$ref": "#/definitions/check" + }, + "description": "Assertions to run against the agent's output and generated files" + } + }, + "definitions": { + "check": { + "type": "object", + "required": ["type", "description"], + "properties": { + "type": { + "type": "string", + "enum": [ + "contains", + "file-exists", + "file-content", + "valid-json", + "command-passes", + "json-schema", + "entity-config", + "agent-config", + "function-def" + ], + "description": "The type of check to perform" + }, + "description": { + "type": "string", + "description": "Human-readable description of what this check validates" + } + }, + "allOf": [ + { + "if": { + "properties": { "type": { "const": "contains" } } + }, + "then": { + "properties": { + "value": { + "type": "string", + "description": "The substring to search for in the agent's output (case-insensitive)" + } + }, + "required": ["value"] + } + }, + { + "if": { + "properties": { "type": { "const": "file-exists" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the file (relative to project directory)" + } + }, + "required": ["filePath"] + } + }, + { + "if": { + "properties": { "type": { "const": "file-content" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the file (relative to project directory)" + }, + "pattern": { + "type": "string", + "description": "Regular expression pattern to match against file content" + }, + "value": { + "type": "string", + "description": "Exact substring to find in file content (alternative to pattern)" + } + }, + "required": ["filePath"] + } + }, + { + "if": { + "properties": { "type": { "const": "valid-json" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the JSON/JSONC file to validate" + } + }, + "required": ["filePath"] + } + }, + { + "if": { + "properties": { "type": { "const": "command-passes" } } + }, + "then": { + "properties": { + "command": { + "type": "string", + "description": "Shell command to run (must exit with code 0 to pass)" + } + }, + "required": ["command"] + } + }, + { + "if": { + "properties": { "type": { "const": "json-schema" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the JSON file to validate (optional, defaults to agent output)" + }, + "schema": { + "type": "object", + "description": "JSON Schema to validate against" + } + }, + "required": ["schema"] + } + }, + { + "if": { + "properties": { "type": { "const": "entity-config" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the entity config file (optional, extracts from output if not specified)" + }, + "target": { + "type": "string", + "enum": ["output", "file"], + "description": "Where to find the entity config" + }, + "expectedValid": { + "type": "boolean", + "default": true, + "description": "Whether the config is expected to be valid" + } + } + } + }, + { + "if": { + "properties": { "type": { "const": "agent-config" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the agent config file (optional, extracts from output if not specified)" + }, + "target": { + "type": "string", + "enum": ["output", "file"], + "description": "Where to find the agent config" + }, + "expectedValid": { + "type": "boolean", + "default": true, + "description": "Whether the config is expected to be valid" + } + } + } + }, + { + "if": { + "properties": { "type": { "const": "function-def" } } + }, + "then": { + "properties": { + "filePath": { + "type": "string", + "description": "Path to the function config file (optional, extracts from output if not specified)" + }, + "target": { + "type": "string", + "enum": ["output", "file"], + "description": "Where to find the function config" + }, + "expectedValid": { + "type": "boolean", + "default": true, + "description": "Whether the config is expected to be valid" + } + } + } + } + ] + } + } +} diff --git a/evals/fixtures/agent-chat-component/eval.json b/evals/fixtures/agent-chat-component/eval.json new file mode 100644 index 0000000..2016947 --- /dev/null +++ b/evals/fixtures/agent-chat-component/eval.json @@ -0,0 +1,43 @@ +{ + "$schema": "../../eval.schema.json", + "name": "agent-chat-component", + "description": "Test creating a React chat component using the Base44 agents SDK", + "prompt": "Create a ChatWindow React component in src/components/ChatWindow.jsx that lets users chat with a support agent. It should: 1) Create a new conversation on mount if none exists, 2) Display messages from the conversation, 3) Allow sending new messages, 4) Subscribe to real-time updates for new messages.", + "expectedSkills": ["base44-sdk"], + "checks": [ + { + "type": "file-exists", + "description": "ChatWindow component created", + "filePath": "src/components/ChatWindow.jsx" + }, + { + "type": "file-content", + "description": "Uses createConversation", + "filePath": "src/components/ChatWindow.jsx", + "pattern": "createConversation" + }, + { + "type": "file-content", + "description": "Uses addMessage", + "filePath": "src/components/ChatWindow.jsx", + "pattern": "addMessage" + }, + { + "type": "file-content", + "description": "Uses subscribeToConversation", + "filePath": "src/components/ChatWindow.jsx", + "pattern": "subscribeToConversation" + }, + { + "type": "file-content", + "description": "References agents module", + "filePath": "src/components/ChatWindow.jsx", + "pattern": "base44\\.agents|agents\\." + }, + { + "type": "contains", + "description": "Mentions agent conversation", + "value": "conversation" + } + ] +} diff --git a/evals/fixtures/agent-with-function/eval.json b/evals/fixtures/agent-with-function/eval.json new file mode 100644 index 0000000..bb17704 --- /dev/null +++ b/evals/fixtures/agent-with-function/eval.json @@ -0,0 +1,52 @@ +{ + "$schema": "../../eval.schema.json", + "name": "agent-with-function", + "description": "Test creating an agent that can call a backend function", + "prompt": "Create an order assistant agent called 'order_assistant' that can help users track and cancel their orders. The agent should have: 1) Read access to the Order entity, 2) Access to a backend function called 'cancel_order' that cancels an order. Also create the cancel_order backend function that accepts order_id and cancellation_reason.", + "expectedSkills": ["base44-cli"], + "checks": [ + { + "type": "file-exists", + "description": "Agent config created", + "filePath": "base44/agents/order_assistant.jsonc" + }, + { + "type": "valid-json", + "description": "Agent config is valid JSON", + "filePath": "base44/agents/order_assistant.jsonc" + }, + { + "type": "file-content", + "description": "Has entity tool config", + "filePath": "base44/agents/order_assistant.jsonc", + "pattern": "\"entity_name\"\\s*:\\s*\"Order\"" + }, + { + "type": "file-content", + "description": "Has function tool config", + "filePath": "base44/agents/order_assistant.jsonc", + "pattern": "\"function_name\"\\s*:\\s*\"cancel_order\"" + }, + { + "type": "file-exists", + "description": "Function config created", + "filePath": "base44/functions/cancel-order/function.jsonc" + }, + { + "type": "file-exists", + "description": "Function entry point created", + "filePath": "base44/functions/cancel-order/index.ts" + }, + { + "type": "file-content", + "description": "Function uses Deno.serve", + "filePath": "base44/functions/cancel-order/index.ts", + "pattern": "Deno\\.serve" + }, + { + "type": "contains", + "description": "Mentions agents", + "value": "agent" + } + ] +} diff --git a/evals/fixtures/create-support-agent/eval.json b/evals/fixtures/create-support-agent/eval.json new file mode 100644 index 0000000..a9809d1 --- /dev/null +++ b/evals/fixtures/create-support-agent/eval.json @@ -0,0 +1,48 @@ +{ + "$schema": "../../eval.schema.json", + "name": "create-support-agent", + "description": "Test creating an AI agent configuration with entity access", + "prompt": "Create a support agent called 'support_agent' that helps users with their orders. The agent should have read access to Order and Customer entities. Give it clear instructions to always look up order details when users ask about their orders.", + "expectedSkills": ["base44-cli"], + "checks": [ + { + "type": "file-exists", + "description": "Agent config file created", + "filePath": "base44/agents/support_agent.jsonc" + }, + { + "type": "valid-json", + "description": "Agent config is valid JSON", + "filePath": "base44/agents/support_agent.jsonc" + }, + { + "type": "file-content", + "description": "Has correct agent name", + "filePath": "base44/agents/support_agent.jsonc", + "pattern": "\"name\"\\s*:\\s*\"support_agent\"" + }, + { + "type": "file-content", + "description": "Has tool_configs with entity access", + "filePath": "base44/agents/support_agent.jsonc", + "pattern": "\"tool_configs\"\\s*:\\s*\\[" + }, + { + "type": "file-content", + "description": "Has Order entity access", + "filePath": "base44/agents/support_agent.jsonc", + "pattern": "\"entity_name\"\\s*:\\s*\"Order\"" + }, + { + "type": "file-content", + "description": "Has instructions field", + "filePath": "base44/agents/support_agent.jsonc", + "pattern": "\"instructions\"\\s*:" + }, + { + "type": "contains", + "description": "Mentions agents push command", + "value": "agents push" + } + ] +} diff --git a/evals/fixtures/existing-app-add-feature/eval.json b/evals/fixtures/existing-app-add-feature/eval.json new file mode 100644 index 0000000..a50a419 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/eval.json @@ -0,0 +1,53 @@ +{ + "$schema": "../../eval.schema.json", + "name": "add-notification-function", + "description": "Test creating a backend function to send order status notification emails", + "prompt": "Create a backend function called 'send-order-notification' that sends an email notification when an order status changes. The function should: 1) Accept order_id and new_status in the request body, 2) Fetch the order and user details using service role, 3) Send an email to the user about their order status update using integrations.Core.SendEmail.", + "expectedSkills": ["base44-cli"], + "checks": [ + { + "type": "file-exists", + "description": "Function directory created", + "filePath": "base44/functions/send-order-notification/function.jsonc" + }, + { + "type": "file-exists", + "description": "Function entry point created", + "filePath": "base44/functions/send-order-notification/index.ts" + }, + { + "type": "valid-json", + "description": "function.jsonc is valid JSON", + "filePath": "base44/functions/send-order-notification/function.jsonc" + }, + { + "type": "file-content", + "description": "Uses Deno.serve", + "filePath": "base44/functions/send-order-notification/index.ts", + "pattern": "Deno\\.serve" + }, + { + "type": "file-content", + "description": "Uses createClientFromRequest", + "filePath": "base44/functions/send-order-notification/index.ts", + "pattern": "createClientFromRequest" + }, + { + "type": "file-content", + "description": "Uses asServiceRole", + "filePath": "base44/functions/send-order-notification/index.ts", + "pattern": "asServiceRole" + }, + { + "type": "file-content", + "description": "Uses SendEmail integration", + "filePath": "base44/functions/send-order-notification/index.ts", + "pattern": "SendEmail|integrations\\.Core" + }, + { + "type": "contains", + "description": "mentions function name", + "value": "send-order-notification" + } + ] +} diff --git a/evals/fixtures/nextjs-todo/eval.json b/evals/fixtures/nextjs-todo/eval.json new file mode 100644 index 0000000..08d852d --- /dev/null +++ b/evals/fixtures/nextjs-todo/eval.json @@ -0,0 +1,52 @@ +{ + "$schema": "../../eval.schema.json", + "name": "nextjs-todo-entities", + "description": "Test creating Todo and Category entities in an existing Next.js Base44 project", + "prompt": "Create Base44 entity schema files in base44/entities/ for a todo app. Create a Todo entity (todo.jsonc) with title (string, required), description (string), completed (boolean, default false), and due_date (string with date format). Also create a Category entity (category.jsonc) with name (string, required) and color (string). The Todo should have a category_id field to reference the Category. Use the standard Base44 entity schema format with name, type, properties, and required fields.", + "expectedSkills": ["base44-cli"], + "checks": [ + { + "type": "file-exists", + "description": "Todo entity file created", + "filePath": "base44/entities/todo.jsonc" + }, + { + "type": "file-exists", + "description": "Category entity file created", + "filePath": "base44/entities/category.jsonc" + }, + { + "type": "valid-json", + "description": "Todo entity is valid JSON", + "filePath": "base44/entities/todo.jsonc" + }, + { + "type": "valid-json", + "description": "Category entity is valid JSON", + "filePath": "base44/entities/category.jsonc" + }, + { + "type": "file-content", + "description": "Todo has title property", + "filePath": "base44/entities/todo.jsonc", + "pattern": "\"title\"\\s*:\\s*\\{" + }, + { + "type": "file-content", + "description": "Todo has completed property", + "filePath": "base44/entities/todo.jsonc", + "pattern": "\"completed\"\\s*:\\s*\\{" + }, + { + "type": "file-content", + "description": "Category has name property", + "filePath": "base44/entities/category.jsonc", + "pattern": "\"name\"\\s*:\\s*\\{" + }, + { + "type": "contains", + "description": "mentions base44 or entities", + "value": "base44" + } + ] +} diff --git a/evals/fixtures/react-task-manager/eval.json b/evals/fixtures/react-task-manager/eval.json new file mode 100644 index 0000000..f0a7c25 --- /dev/null +++ b/evals/fixtures/react-task-manager/eval.json @@ -0,0 +1,38 @@ +{ + "$schema": "../../eval.schema.json", + "name": "react-task-manager-sdk", + "description": "Test implementing a TaskList component using Base44 SDK to fetch and display tasks", + "prompt": "Implement the TaskList component in src/components/TaskList.tsx. It should fetch all tasks using the Base44 SDK (base44.entities.Task.list()), display them in a list showing title, status, and due_date, and allow marking tasks as completed by clicking on them (using base44.entities.Task.update()).", + "expectedSkills": ["base44-sdk"], + "checks": [ + { + "type": "file-content", + "description": "Uses entities.Task.list()", + "filePath": "src/components/TaskList.tsx", + "pattern": "entities\\.Task\\.list\\(" + }, + { + "type": "file-content", + "description": "Uses entities.Task.update()", + "filePath": "src/components/TaskList.tsx", + "pattern": "entities\\.Task\\.update\\(" + }, + { + "type": "file-content", + "description": "Imports base44 client", + "filePath": "src/components/TaskList.tsx", + "pattern": "import.*base44|from ['\"].*base44" + }, + { + "type": "file-content", + "description": "Uses React hooks for state", + "filePath": "src/components/TaskList.tsx", + "pattern": "useState|useEffect" + }, + { + "type": "contains", + "description": "Output mentions SDK or entities", + "value": "entities" + } + ] +} diff --git a/evals/fixtures/skill-check/eval.json b/evals/fixtures/skill-check/eval.json new file mode 100644 index 0000000..91d591c --- /dev/null +++ b/evals/fixtures/skill-check/eval.json @@ -0,0 +1,19 @@ +{ + "$schema": "../../eval.schema.json", + "name": "skill-check", + "description": "Simple test to verify Claude can respond about Base44", + "prompt": "What is Base44? How do I create an entity?", + "expectedSkills": [], + "checks": [ + { + "type": "contains", + "description": "mentions base44", + "value": "base44" + }, + { + "type": "contains", + "description": "mentions entities", + "value": "entit" + } + ] +} From 533868ad9aa17a04c4c2bdcbdd00cb016b9ef496 Mon Sep 17 00:00:00 2001 From: Gal Elmalah Date: Wed, 4 Feb 2026 10:39:19 +0200 Subject: [PATCH 2/3] feat(evals): add evaluation framework with fixtures and multi-fixture support Adds a complete evaluation system for testing skills: - CLI runner with support for specific fixture selection via positional args - Multiple check types (file-exists, file-content, contains, json-schema, etc.) - Markdown report generation - Claude Code agent integration - 7 test fixtures covering various Base44 SDK scenarios Co-Authored-By: Claude Opus 4.5 --- evals/.gitignore | 14 + evals/fixtures/agent-chat-component/AGENTS.md | 14 + .../project/.claude/settings.json | 6 + .../agent-chat-component/project/CLAUDE.md | 18 + .../project/base44/agents/support_agent.jsonc | 5 + .../project/base44/config.jsonc | 4 + .../agent-chat-component/project/package.json | 10 + .../project/src/api/base44Client.js | 5 + .../skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ evals/fixtures/agent-with-function/AGENTS.md | 15 + .../project/.claude/settings.json | 6 + .../agent-with-function/project/CLAUDE.md | 18 + .../project/base44/config.jsonc | 4 + .../agent-with-function/project/package.json | 9 + .../skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ evals/fixtures/create-support-agent/AGENTS.md | 14 + .../project/.claude/settings.json | 6 + .../create-support-agent/project/CLAUDE.md | 18 + .../project/base44/config.jsonc | 4 + .../create-support-agent/project/package.json | 9 + .../skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ .../existing-app-add-feature/AGENTS.md | 21 + .../project/.claude/settings.json | 6 + .../project/CLAUDE.md | 18 + .../project/base44/config.jsonc | 6 + .../project/base44/entities/order.jsonc | 29 + .../project/base44/entities/user.jsonc | 23 + .../project/package.json | 24 + .../project/src/lib/base44.ts | 5 + .../skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ evals/fixtures/nextjs-todo/AGENTS.md | 14 + .../nextjs-todo/project/.claude/settings.json | 6 + evals/fixtures/nextjs-todo/project/CLAUDE.md | 18 + .../nextjs-todo/project/base44/config.jsonc | 10 + .../nextjs-todo/project/next.config.js | 4 + .../fixtures/nextjs-todo/project/package.json | 26 + .../nextjs-todo/project/src/app/layout.tsx | 16 + .../nextjs-todo/project/src/app/page.tsx | 8 + .../nextjs-todo/project/src/lib/base44.ts | 5 + .../nextjs-todo/project/tsconfig.json | 26 + .../nextjs-todo/skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../nextjs-todo/skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ evals/fixtures/react-task-manager/AGENTS.md | 20 + .../project/.claude/settings.json | 6 + .../react-task-manager/project/CLAUDE.md | 18 + .../project/base44/config.jsonc | 10 + .../project/base44/entities/task.jsonc | 29 + .../react-task-manager/project/index.html | 12 + .../react-task-manager/project/package.json | 30 + .../react-task-manager/project/src/App.tsx | 12 + .../project/src/components/TaskList.tsx | 9 + .../project/src/lib/base44.ts | 5 + .../react-task-manager/project/src/main.tsx | 9 + .../react-task-manager/project/tsconfig.json | 21 + .../project/tsconfig.node.json | 11 + .../react-task-manager/project/vite.config.ts | 6 + .../skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ evals/fixtures/skill-check/AGENTS.md | 8 + .../skill-check/project/.claude/settings.json | 6 + evals/fixtures/skill-check/project/CLAUDE.md | 18 + .../skill-check/project/base44/config.jsonc | 4 + .../fixtures/skill-check/project/package.json | 4 + .../skill-check/skills/base44-cli/SKILL.md | 384 ++++++++ .../base44-cli/references/agents-pull.md | 73 ++ .../base44-cli/references/agents-push.md | 156 +++ .../base44-cli/references/auth-login.md | 54 ++ .../base44-cli/references/auth-logout.md | 32 + .../base44-cli/references/auth-whoami.md | 37 + .../skills/base44-cli/references/create.md | 111 +++ .../skills/base44-cli/references/dashboard.md | 35 + .../skills/base44-cli/references/deploy.md | 86 ++ .../base44-cli/references/entities-create.md | 552 +++++++++++ .../base44-cli/references/entities-push.md | 71 ++ .../base44-cli/references/functions-create.md | 229 +++++ .../base44-cli/references/functions-deploy.md | 78 ++ .../skills/base44-cli/references/link.md | 81 ++ .../base44-cli/references/rls-examples.md | 463 +++++++++ .../base44-cli/references/site-deploy.md | 118 +++ .../skill-check/skills/base44-sdk/SKILL.md | 296 ++++++ .../base44-sdk/references/QUICK_REFERENCE.md | 155 +++ .../skills/base44-sdk/references/analytics.md | 113 +++ .../skills/base44-sdk/references/app-logs.md | 78 ++ .../skills/base44-sdk/references/auth.md | 761 +++++++++++++++ .../base44-sdk/references/base44-agents.md | 364 +++++++ .../skills/base44-sdk/references/client.md | 257 +++++ .../base44-sdk/references/connectors.md | 113 +++ .../skills/base44-sdk/references/entities.md | 253 +++++ .../skills/base44-sdk/references/functions.md | 216 +++++ .../base44-sdk/references/integrations.md | 377 ++++++++ .../skills/base44-sdk/references/users.md | 82 ++ evals/package-lock.json | 891 ++++++++++++++++++ evals/package.json | 24 + evals/src/agents/claude-code.ts | 147 +++ evals/src/agents/interface.ts | 27 + evals/src/agents/mock.ts | 145 +++ evals/src/checks/agent-config.ts | 105 +++ evals/src/checks/command-passes.ts | 53 ++ evals/src/checks/contains.ts | 31 + evals/src/checks/entity-config.ts | 128 +++ evals/src/checks/file-content.ts | 70 ++ evals/src/checks/file-exists.ts | 42 + evals/src/checks/function-def.ts | 97 ++ evals/src/checks/interface.ts | 45 + evals/src/checks/json-schema.ts | 133 +++ evals/src/checks/valid-json.ts | 49 + evals/src/compare-runner.ts | 200 ++++ evals/src/compare.ts | 264 ++++++ evals/src/experiments.ts | 60 ++ evals/src/index.ts | 145 +++ evals/src/reporters/comparison-markdown.ts | 202 ++++ evals/src/reporters/markdown.ts | 139 +++ evals/src/runner.ts | 278 ++++++ evals/src/setup.ts | 156 +++ evals/src/types.ts | 111 +++ evals/tsconfig.json | 16 + 276 files changed, 43615 insertions(+) create mode 100644 evals/.gitignore create mode 100644 evals/fixtures/agent-chat-component/AGENTS.md create mode 100644 evals/fixtures/agent-chat-component/project/.claude/settings.json create mode 100644 evals/fixtures/agent-chat-component/project/CLAUDE.md create mode 100644 evals/fixtures/agent-chat-component/project/base44/agents/support_agent.jsonc create mode 100644 evals/fixtures/agent-chat-component/project/base44/config.jsonc create mode 100644 evals/fixtures/agent-chat-component/project/package.json create mode 100644 evals/fixtures/agent-chat-component/project/src/api/base44Client.js create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/agent-chat-component/skills/base44-sdk/references/users.md create mode 100644 evals/fixtures/agent-with-function/AGENTS.md create mode 100644 evals/fixtures/agent-with-function/project/.claude/settings.json create mode 100644 evals/fixtures/agent-with-function/project/CLAUDE.md create mode 100644 evals/fixtures/agent-with-function/project/base44/config.jsonc create mode 100644 evals/fixtures/agent-with-function/project/package.json create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/agent-with-function/skills/base44-sdk/references/users.md create mode 100644 evals/fixtures/create-support-agent/AGENTS.md create mode 100644 evals/fixtures/create-support-agent/project/.claude/settings.json create mode 100644 evals/fixtures/create-support-agent/project/CLAUDE.md create mode 100644 evals/fixtures/create-support-agent/project/base44/config.jsonc create mode 100644 evals/fixtures/create-support-agent/project/package.json create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/create-support-agent/skills/base44-sdk/references/users.md create mode 100644 evals/fixtures/existing-app-add-feature/AGENTS.md create mode 100644 evals/fixtures/existing-app-add-feature/project/.claude/settings.json create mode 100644 evals/fixtures/existing-app-add-feature/project/CLAUDE.md create mode 100644 evals/fixtures/existing-app-add-feature/project/base44/config.jsonc create mode 100644 evals/fixtures/existing-app-add-feature/project/base44/entities/order.jsonc create mode 100644 evals/fixtures/existing-app-add-feature/project/base44/entities/user.jsonc create mode 100644 evals/fixtures/existing-app-add-feature/project/package.json create mode 100644 evals/fixtures/existing-app-add-feature/project/src/lib/base44.ts create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/users.md create mode 100644 evals/fixtures/nextjs-todo/AGENTS.md create mode 100644 evals/fixtures/nextjs-todo/project/.claude/settings.json create mode 100644 evals/fixtures/nextjs-todo/project/CLAUDE.md create mode 100644 evals/fixtures/nextjs-todo/project/base44/config.jsonc create mode 100644 evals/fixtures/nextjs-todo/project/next.config.js create mode 100644 evals/fixtures/nextjs-todo/project/package.json create mode 100644 evals/fixtures/nextjs-todo/project/src/app/layout.tsx create mode 100644 evals/fixtures/nextjs-todo/project/src/app/page.tsx create mode 100644 evals/fixtures/nextjs-todo/project/src/lib/base44.ts create mode 100644 evals/fixtures/nextjs-todo/project/tsconfig.json create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/nextjs-todo/skills/base44-sdk/references/users.md create mode 100644 evals/fixtures/react-task-manager/AGENTS.md create mode 100644 evals/fixtures/react-task-manager/project/.claude/settings.json create mode 100644 evals/fixtures/react-task-manager/project/CLAUDE.md create mode 100644 evals/fixtures/react-task-manager/project/base44/config.jsonc create mode 100644 evals/fixtures/react-task-manager/project/base44/entities/task.jsonc create mode 100644 evals/fixtures/react-task-manager/project/index.html create mode 100644 evals/fixtures/react-task-manager/project/package.json create mode 100644 evals/fixtures/react-task-manager/project/src/App.tsx create mode 100644 evals/fixtures/react-task-manager/project/src/components/TaskList.tsx create mode 100644 evals/fixtures/react-task-manager/project/src/lib/base44.ts create mode 100644 evals/fixtures/react-task-manager/project/src/main.tsx create mode 100644 evals/fixtures/react-task-manager/project/tsconfig.json create mode 100644 evals/fixtures/react-task-manager/project/tsconfig.node.json create mode 100644 evals/fixtures/react-task-manager/project/vite.config.ts create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/react-task-manager/skills/base44-sdk/references/users.md create mode 100644 evals/fixtures/skill-check/AGENTS.md create mode 100644 evals/fixtures/skill-check/project/.claude/settings.json create mode 100644 evals/fixtures/skill-check/project/CLAUDE.md create mode 100644 evals/fixtures/skill-check/project/base44/config.jsonc create mode 100644 evals/fixtures/skill-check/project/package.json create mode 100644 evals/fixtures/skill-check/skills/base44-cli/SKILL.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/agents-pull.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/agents-push.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/auth-login.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/auth-logout.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/auth-whoami.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/create.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/dashboard.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/deploy.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/entities-create.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/entities-push.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/functions-create.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/functions-deploy.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/link.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/rls-examples.md create mode 100644 evals/fixtures/skill-check/skills/base44-cli/references/site-deploy.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/SKILL.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/QUICK_REFERENCE.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/analytics.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/app-logs.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/auth.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/base44-agents.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/client.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/connectors.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/entities.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/functions.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/integrations.md create mode 100644 evals/fixtures/skill-check/skills/base44-sdk/references/users.md create mode 100644 evals/package-lock.json create mode 100644 evals/package.json create mode 100644 evals/src/agents/claude-code.ts create mode 100644 evals/src/agents/interface.ts create mode 100644 evals/src/agents/mock.ts create mode 100644 evals/src/checks/agent-config.ts create mode 100644 evals/src/checks/command-passes.ts create mode 100644 evals/src/checks/contains.ts create mode 100644 evals/src/checks/entity-config.ts create mode 100644 evals/src/checks/file-content.ts create mode 100644 evals/src/checks/file-exists.ts create mode 100644 evals/src/checks/function-def.ts create mode 100644 evals/src/checks/interface.ts create mode 100644 evals/src/checks/json-schema.ts create mode 100644 evals/src/checks/valid-json.ts create mode 100644 evals/src/compare-runner.ts create mode 100644 evals/src/compare.ts create mode 100644 evals/src/experiments.ts create mode 100644 evals/src/index.ts create mode 100644 evals/src/reporters/comparison-markdown.ts create mode 100644 evals/src/reporters/markdown.ts create mode 100644 evals/src/runner.ts create mode 100644 evals/src/setup.ts create mode 100644 evals/src/types.ts create mode 100644 evals/tsconfig.json diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 0000000..71cefbc --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,14 @@ +# Dependencies +node_modules/ + +# Build output +dist/ + +# Results (generated reports) +results/*.md + +# Copied skills in fixtures (synced from parent /skills directory by setup script) +fixtures/*/project/skills/ + +# OS files +.DS_Store diff --git a/evals/fixtures/agent-chat-component/AGENTS.md b/evals/fixtures/agent-chat-component/AGENTS.md new file mode 100644 index 0000000..8ee4cb6 --- /dev/null +++ b/evals/fixtures/agent-chat-component/AGENTS.md @@ -0,0 +1,14 @@ +# Agent Chat Component Fixture + +This fixture tests implementing a chat interface using the Base44 agents SDK. + +## Project Context + +This is a React app with Base44 SDK configured. A support_agent is already set up in the backend. + +## Expected Outcome + +Create a ChatWindow component that uses the Base44 agents SDK to: +- Create conversations +- Send and receive messages +- Subscribe to real-time updates diff --git a/evals/fixtures/agent-chat-component/project/.claude/settings.json b/evals/fixtures/agent-chat-component/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/agent-chat-component/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/agent-chat-component/project/CLAUDE.md b/evals/fixtures/agent-chat-component/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/agent-chat-component/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/agent-chat-component/project/base44/agents/support_agent.jsonc b/evals/fixtures/agent-chat-component/project/base44/agents/support_agent.jsonc new file mode 100644 index 0000000..b81a0aa --- /dev/null +++ b/evals/fixtures/agent-chat-component/project/base44/agents/support_agent.jsonc @@ -0,0 +1,5 @@ +{ + "name": "support_agent", + "description": "A helpful support assistant", + "instructions": "Help users with their questions" +} diff --git a/evals/fixtures/agent-chat-component/project/base44/config.jsonc b/evals/fixtures/agent-chat-component/project/base44/config.jsonc new file mode 100644 index 0000000..fa444eb --- /dev/null +++ b/evals/fixtures/agent-chat-component/project/base44/config.jsonc @@ -0,0 +1,4 @@ +{ + "name": "Chat App", + "description": "App with AI agent chat support" +} diff --git a/evals/fixtures/agent-chat-component/project/package.json b/evals/fixtures/agent-chat-component/project/package.json new file mode 100644 index 0000000..c9dd3fc --- /dev/null +++ b/evals/fixtures/agent-chat-component/project/package.json @@ -0,0 +1,10 @@ +{ + "name": "chat-app", + "version": "1.0.0", + "type": "module", + "dependencies": { + "@base44/sdk": "latest", + "react": "^18.2.0", + "react-dom": "^18.2.0" + } +} diff --git a/evals/fixtures/agent-chat-component/project/src/api/base44Client.js b/evals/fixtures/agent-chat-component/project/src/api/base44Client.js new file mode 100644 index 0000000..dddb0d8 --- /dev/null +++ b/evals/fixtures/agent-chat-component/project/src/api/base44Client.js @@ -0,0 +1,5 @@ +import { createClient } from '@base44/sdk'; + +export const base44 = createClient({ + appId: 'demo-app' +}); diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/SKILL.md b/evals/fixtures/agent-chat-component/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-pull.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-push.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-login.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-logout.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/create.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/dashboard.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/deploy.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-create.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-push.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-create.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/link.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/rls-examples.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/agent-chat-component/skills/base44-cli/references/site-deploy.md b/evals/fixtures/agent-chat-component/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/SKILL.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/analytics.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/app-logs.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/auth.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/client.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/connectors.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/entities.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/functions.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/integrations.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/agent-chat-component/skills/base44-sdk/references/users.md b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/agent-chat-component/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/fixtures/agent-with-function/AGENTS.md b/evals/fixtures/agent-with-function/AGENTS.md new file mode 100644 index 0000000..35ba3e2 --- /dev/null +++ b/evals/fixtures/agent-with-function/AGENTS.md @@ -0,0 +1,15 @@ +# Agent with Function Fixture + +This fixture tests creating an AI agent that has access to a backend function. + +## Project Context + +This is an e-commerce app that needs an order assistant agent capable of: +- Reading order information +- Canceling orders via a backend function + +## Expected Outcome + +Create both: +1. An agent config at `base44/agents/order_assistant.jsonc` with both entity and function tool access +2. A backend function at `base44/functions/cancel-order/` that handles order cancellation diff --git a/evals/fixtures/agent-with-function/project/.claude/settings.json b/evals/fixtures/agent-with-function/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/agent-with-function/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/agent-with-function/project/CLAUDE.md b/evals/fixtures/agent-with-function/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/agent-with-function/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/agent-with-function/project/base44/config.jsonc b/evals/fixtures/agent-with-function/project/base44/config.jsonc new file mode 100644 index 0000000..5e061a3 --- /dev/null +++ b/evals/fixtures/agent-with-function/project/base44/config.jsonc @@ -0,0 +1,4 @@ +{ + "name": "Order Management App", + "description": "E-commerce order management with AI assistant" +} diff --git a/evals/fixtures/agent-with-function/project/package.json b/evals/fixtures/agent-with-function/project/package.json new file mode 100644 index 0000000..29d668d --- /dev/null +++ b/evals/fixtures/agent-with-function/project/package.json @@ -0,0 +1,9 @@ +{ + "name": "order-management", + "version": "1.0.0", + "type": "module", + "dependencies": { + "@base44/sdk": "latest", + "react": "^18.2.0" + } +} diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/SKILL.md b/evals/fixtures/agent-with-function/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/agents-pull.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/agents-push.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-login.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-logout.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/create.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/dashboard.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/deploy.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/entities-create.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/entities-push.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/functions-create.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/link.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/rls-examples.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/agent-with-function/skills/base44-cli/references/site-deploy.md b/evals/fixtures/agent-with-function/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/SKILL.md b/evals/fixtures/agent-with-function/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/analytics.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/app-logs.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/auth.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/client.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/connectors.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/entities.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/functions.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/integrations.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/agent-with-function/skills/base44-sdk/references/users.md b/evals/fixtures/agent-with-function/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/agent-with-function/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/fixtures/create-support-agent/AGENTS.md b/evals/fixtures/create-support-agent/AGENTS.md new file mode 100644 index 0000000..67387aa --- /dev/null +++ b/evals/fixtures/create-support-agent/AGENTS.md @@ -0,0 +1,14 @@ +# Support Agent Fixture + +This fixture tests creating an AI agent configuration file for a support assistant. + +## Project Context + +This is an existing Base44 project for an e-commerce app. It has Order and Customer entities already defined. + +## Expected Outcome + +Create an agent config file at `base44/agents/support_agent.jsonc` that: +- Uses the correct agent schema format +- Has `tool_configs` with entity access (not the old `tools` format) +- Includes clear instructions for the agent behavior diff --git a/evals/fixtures/create-support-agent/project/.claude/settings.json b/evals/fixtures/create-support-agent/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/create-support-agent/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/create-support-agent/project/CLAUDE.md b/evals/fixtures/create-support-agent/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/create-support-agent/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/create-support-agent/project/base44/config.jsonc b/evals/fixtures/create-support-agent/project/base44/config.jsonc new file mode 100644 index 0000000..463a0c4 --- /dev/null +++ b/evals/fixtures/create-support-agent/project/base44/config.jsonc @@ -0,0 +1,4 @@ +{ + "name": "E-Commerce App", + "description": "Online store with order management" +} diff --git a/evals/fixtures/create-support-agent/project/package.json b/evals/fixtures/create-support-agent/project/package.json new file mode 100644 index 0000000..307831b --- /dev/null +++ b/evals/fixtures/create-support-agent/project/package.json @@ -0,0 +1,9 @@ +{ + "name": "ecommerce-app", + "version": "1.0.0", + "type": "module", + "dependencies": { + "@base44/sdk": "latest", + "react": "^18.2.0" + } +} diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/SKILL.md b/evals/fixtures/create-support-agent/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/agents-pull.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/agents-push.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-login.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-logout.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/create.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/dashboard.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/deploy.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/entities-create.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/entities-push.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/functions-create.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/link.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/rls-examples.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/create-support-agent/skills/base44-cli/references/site-deploy.md b/evals/fixtures/create-support-agent/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/SKILL.md b/evals/fixtures/create-support-agent/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/analytics.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/app-logs.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/auth.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/client.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/connectors.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/entities.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/functions.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/integrations.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/create-support-agent/skills/base44-sdk/references/users.md b/evals/fixtures/create-support-agent/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/create-support-agent/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/AGENTS.md b/evals/fixtures/existing-app-add-feature/AGENTS.md new file mode 100644 index 0000000..e3696a9 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/AGENTS.md @@ -0,0 +1,21 @@ +# Notification App + +An application that handles orders and sends email notifications. + +## Project Structure + +- `src/` - Frontend source code +- `src/lib/base44.ts` - Base44 SDK client +- `base44/` - Base44 configuration +- `base44/entities/` - Entity definitions (User, Order) +- `base44/functions/` - Backend functions (to be created) + +## Existing Entities + +- **User**: email, name, notification_preferences +- **Order**: user_id, items, status, total + +## Available Skills + +- **base44-cli**: CLI for managing Base44 entities, functions, and deployment +- **base44-sdk**: SDK for building features with Base44 APIs diff --git a/evals/fixtures/existing-app-add-feature/project/.claude/settings.json b/evals/fixtures/existing-app-add-feature/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/existing-app-add-feature/project/CLAUDE.md b/evals/fixtures/existing-app-add-feature/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/existing-app-add-feature/project/base44/config.jsonc b/evals/fixtures/existing-app-add-feature/project/base44/config.jsonc new file mode 100644 index 0000000..ebca7e2 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/base44/config.jsonc @@ -0,0 +1,6 @@ +{ + "name": "Notification App", + "description": "An application with email notifications", + "entitiesDir": "./entities", + "functionsDir": "./functions" +} diff --git a/evals/fixtures/existing-app-add-feature/project/base44/entities/order.jsonc b/evals/fixtures/existing-app-add-feature/project/base44/entities/order.jsonc new file mode 100644 index 0000000..48a6194 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/base44/entities/order.jsonc @@ -0,0 +1,29 @@ +{ + "name": "Order", + "type": "object", + "properties": { + "user_id": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "quantity": { "type": "number" }, + "price": { "type": "number" } + } + } + }, + "status": { + "type": "string", + "enum": ["pending", "processing", "shipped", "delivered"], + "default": "pending" + }, + "total": { + "type": "number" + } + }, + "required": ["user_id", "items", "total"] +} diff --git a/evals/fixtures/existing-app-add-feature/project/base44/entities/user.jsonc b/evals/fixtures/existing-app-add-feature/project/base44/entities/user.jsonc new file mode 100644 index 0000000..eb7cbe7 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/base44/entities/user.jsonc @@ -0,0 +1,23 @@ +{ + "name": "User", + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email" + }, + "name": { + "type": "string" + }, + "notification_preferences": { + "type": "object", + "properties": { + "email_enabled": { + "type": "boolean", + "default": true + } + } + } + }, + "required": ["email", "name"] +} diff --git a/evals/fixtures/existing-app-add-feature/project/package.json b/evals/fixtures/existing-app-add-feature/project/package.json new file mode 100644 index 0000000..49e5863 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/package.json @@ -0,0 +1,24 @@ +{ + "name": "notification-app", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "lint": "eslint . --ext ts,tsx" + }, + "dependencies": { + "@base44/sdk": "^1.0.0", + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.66", + "@types/react-dom": "^18.2.22", + "base44": "^1.0.0", + "eslint": "^8.57.0", + "typescript": "^5.2.2", + "vite": "^5.2.0" + } +} diff --git a/evals/fixtures/existing-app-add-feature/project/src/lib/base44.ts b/evals/fixtures/existing-app-add-feature/project/src/lib/base44.ts new file mode 100644 index 0000000..9606ca2 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/project/src/lib/base44.ts @@ -0,0 +1,5 @@ +import { createClient } from '@base44/sdk'; + +export const base44 = createClient({ + appId: import.meta.env.VITE_BASE44_APP_ID, +}); diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/SKILL.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-pull.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-push.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-login.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-logout.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/create.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/dashboard.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/deploy.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-create.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-push.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-create.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/link.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/rls-examples.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/site-deploy.md b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/SKILL.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/analytics.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/app-logs.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/auth.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/client.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/connectors.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/entities.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/functions.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/integrations.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/users.md b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/existing-app-add-feature/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/fixtures/nextjs-todo/AGENTS.md b/evals/fixtures/nextjs-todo/AGENTS.md new file mode 100644 index 0000000..37e2902 --- /dev/null +++ b/evals/fixtures/nextjs-todo/AGENTS.md @@ -0,0 +1,14 @@ +# Todo App + +A Next.js application using Base44 for backend services. + +## Project Structure + +- `src/app/` - Next.js App Router pages +- `src/lib/base44.ts` - Base44 SDK client +- `base44/` - Base44 configuration and entities + +## Available Skills + +- **base44-cli**: CLI for managing Base44 entities, functions, and deployment +- **base44-sdk**: SDK for building features with Base44 APIs (entities, auth, etc.) diff --git a/evals/fixtures/nextjs-todo/project/.claude/settings.json b/evals/fixtures/nextjs-todo/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/nextjs-todo/project/CLAUDE.md b/evals/fixtures/nextjs-todo/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/nextjs-todo/project/base44/config.jsonc b/evals/fixtures/nextjs-todo/project/base44/config.jsonc new file mode 100644 index 0000000..11f15ea --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/base44/config.jsonc @@ -0,0 +1,10 @@ +{ + "name": "Todo App", + "description": "A simple todo application", + "entitiesDir": "./entities", + "functionsDir": "./functions", + "site": { + "buildCommand": "npm run build", + "outputDirectory": ".next" + } +} diff --git a/evals/fixtures/nextjs-todo/project/next.config.js b/evals/fixtures/nextjs-todo/project/next.config.js new file mode 100644 index 0000000..658404a --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/next.config.js @@ -0,0 +1,4 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = {}; + +module.exports = nextConfig; diff --git a/evals/fixtures/nextjs-todo/project/package.json b/evals/fixtures/nextjs-todo/project/package.json new file mode 100644 index 0000000..fc3e965 --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/package.json @@ -0,0 +1,26 @@ +{ + "name": "todo-app", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint" + }, + "dependencies": { + "@base44/sdk": "^1.0.0", + "next": "14.2.5", + "react": "^18", + "react-dom": "^18" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^18", + "@types/react-dom": "^18", + "base44": "^1.0.0", + "eslint": "^8", + "eslint-config-next": "14.2.5", + "typescript": "^5" + } +} diff --git a/evals/fixtures/nextjs-todo/project/src/app/layout.tsx b/evals/fixtures/nextjs-todo/project/src/app/layout.tsx new file mode 100644 index 0000000..18de1e9 --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/src/app/layout.tsx @@ -0,0 +1,16 @@ +export const metadata = { + title: 'Todo App', + description: 'A simple todo application', +}; + +export default function RootLayout({ + children, +}: { + children: React.ReactNode; +}) { + return ( + + {children} + + ); +} diff --git a/evals/fixtures/nextjs-todo/project/src/app/page.tsx b/evals/fixtures/nextjs-todo/project/src/app/page.tsx new file mode 100644 index 0000000..4f4b4fc --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/src/app/page.tsx @@ -0,0 +1,8 @@ +export default function Home() { + return ( +
+

Todo App

+

Welcome to your todo application.

+
+ ); +} diff --git a/evals/fixtures/nextjs-todo/project/src/lib/base44.ts b/evals/fixtures/nextjs-todo/project/src/lib/base44.ts new file mode 100644 index 0000000..725ac2c --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/src/lib/base44.ts @@ -0,0 +1,5 @@ +import { createClient } from '@base44/sdk'; + +export const base44 = createClient({ + appId: process.env.NEXT_PUBLIC_BASE44_APP_ID!, +}); diff --git a/evals/fixtures/nextjs-todo/project/tsconfig.json b/evals/fixtures/nextjs-todo/project/tsconfig.json new file mode 100644 index 0000000..7b28589 --- /dev/null +++ b/evals/fixtures/nextjs-todo/project/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/SKILL.md b/evals/fixtures/nextjs-todo/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-pull.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-push.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-login.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-logout.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/create.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/dashboard.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/deploy.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-create.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-push.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-create.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/link.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/rls-examples.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/nextjs-todo/skills/base44-cli/references/site-deploy.md b/evals/fixtures/nextjs-todo/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/SKILL.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/analytics.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/app-logs.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/auth.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/client.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/connectors.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/entities.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/functions.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/integrations.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/nextjs-todo/skills/base44-sdk/references/users.md b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/nextjs-todo/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/fixtures/react-task-manager/AGENTS.md b/evals/fixtures/react-task-manager/AGENTS.md new file mode 100644 index 0000000..e688fdc --- /dev/null +++ b/evals/fixtures/react-task-manager/AGENTS.md @@ -0,0 +1,20 @@ +# Task Manager + +A React + Vite application using Base44 for backend services. + +## Project Structure + +- `src/` - React source code +- `src/components/` - React components +- `src/lib/base44.ts` - Base44 SDK client +- `base44/` - Base44 configuration +- `base44/entities/` - Entity definitions (Task already exists) + +## Existing Entities + +- **Task**: title, description, status (pending/in_progress/completed), priority, due_date + +## Available Skills + +- **base44-cli**: CLI for managing Base44 entities, functions, and deployment +- **base44-sdk**: SDK for building features with Base44 APIs (entities, auth, etc.) diff --git a/evals/fixtures/react-task-manager/project/.claude/settings.json b/evals/fixtures/react-task-manager/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/react-task-manager/project/CLAUDE.md b/evals/fixtures/react-task-manager/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/react-task-manager/project/base44/config.jsonc b/evals/fixtures/react-task-manager/project/base44/config.jsonc new file mode 100644 index 0000000..1a7bf1b --- /dev/null +++ b/evals/fixtures/react-task-manager/project/base44/config.jsonc @@ -0,0 +1,10 @@ +{ + "name": "Task Manager", + "description": "A task management application", + "entitiesDir": "./entities", + "functionsDir": "./functions", + "site": { + "buildCommand": "npm run build", + "outputDirectory": "dist" + } +} diff --git a/evals/fixtures/react-task-manager/project/base44/entities/task.jsonc b/evals/fixtures/react-task-manager/project/base44/entities/task.jsonc new file mode 100644 index 0000000..aaa2cb5 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/base44/entities/task.jsonc @@ -0,0 +1,29 @@ +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["pending", "in_progress", "completed"], + "default": "pending" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium" + }, + "due_date": { + "type": "string", + "format": "date" + } + }, + "required": ["title"] +} diff --git a/evals/fixtures/react-task-manager/project/index.html b/evals/fixtures/react-task-manager/project/index.html new file mode 100644 index 0000000..80927a8 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/index.html @@ -0,0 +1,12 @@ + + + + + + Task Manager + + +
+ + + diff --git a/evals/fixtures/react-task-manager/project/package.json b/evals/fixtures/react-task-manager/project/package.json new file mode 100644 index 0000000..f4cedd3 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/package.json @@ -0,0 +1,30 @@ +{ + "name": "task-manager", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "preview": "vite preview" + }, + "dependencies": { + "@base44/sdk": "^1.0.0", + "react": "^18.2.0", + "react-dom": "^18.2.0" + }, + "devDependencies": { + "@types/react": "^18.2.66", + "@types/react-dom": "^18.2.22", + "@typescript-eslint/eslint-plugin": "^7.2.0", + "@typescript-eslint/parser": "^7.2.0", + "@vitejs/plugin-react": "^4.2.1", + "base44": "^1.0.0", + "eslint": "^8.57.0", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.6", + "typescript": "^5.2.2", + "vite": "^5.2.0" + } +} diff --git a/evals/fixtures/react-task-manager/project/src/App.tsx b/evals/fixtures/react-task-manager/project/src/App.tsx new file mode 100644 index 0000000..0dbffa6 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/src/App.tsx @@ -0,0 +1,12 @@ +import { TaskList } from './components/TaskList'; + +function App() { + return ( +
+

Task Manager

+ +
+ ); +} + +export default App; diff --git a/evals/fixtures/react-task-manager/project/src/components/TaskList.tsx b/evals/fixtures/react-task-manager/project/src/components/TaskList.tsx new file mode 100644 index 0000000..e62769e --- /dev/null +++ b/evals/fixtures/react-task-manager/project/src/components/TaskList.tsx @@ -0,0 +1,9 @@ +// TODO: Implement task list component that fetches and displays tasks +export function TaskList() { + return ( +
+

Tasks

+

Task list will be displayed here

+
+ ); +} diff --git a/evals/fixtures/react-task-manager/project/src/lib/base44.ts b/evals/fixtures/react-task-manager/project/src/lib/base44.ts new file mode 100644 index 0000000..9606ca2 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/src/lib/base44.ts @@ -0,0 +1,5 @@ +import { createClient } from '@base44/sdk'; + +export const base44 = createClient({ + appId: import.meta.env.VITE_BASE44_APP_ID, +}); diff --git a/evals/fixtures/react-task-manager/project/src/main.tsx b/evals/fixtures/react-task-manager/project/src/main.tsx new file mode 100644 index 0000000..e5775c0 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/src/main.tsx @@ -0,0 +1,9 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import App from './App.tsx'; + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + , +); diff --git a/evals/fixtures/react-task-manager/project/tsconfig.json b/evals/fixtures/react-task-manager/project/tsconfig.json new file mode 100644 index 0000000..3934b8f --- /dev/null +++ b/evals/fixtures/react-task-manager/project/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/evals/fixtures/react-task-manager/project/tsconfig.node.json b/evals/fixtures/react-task-manager/project/tsconfig.node.json new file mode 100644 index 0000000..97ede7e --- /dev/null +++ b/evals/fixtures/react-task-manager/project/tsconfig.node.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true, + "strict": true + }, + "include": ["vite.config.ts"] +} diff --git a/evals/fixtures/react-task-manager/project/vite.config.ts b/evals/fixtures/react-task-manager/project/vite.config.ts new file mode 100644 index 0000000..0466183 --- /dev/null +++ b/evals/fixtures/react-task-manager/project/vite.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +export default defineConfig({ + plugins: [react()], +}); diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/SKILL.md b/evals/fixtures/react-task-manager/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/agents-pull.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/agents-push.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-login.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-logout.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/create.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/dashboard.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/deploy.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/entities-create.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/entities-push.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/functions-create.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/link.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/rls-examples.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/react-task-manager/skills/base44-cli/references/site-deploy.md b/evals/fixtures/react-task-manager/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/SKILL.md b/evals/fixtures/react-task-manager/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/analytics.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/app-logs.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/auth.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/client.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/connectors.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/entities.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/functions.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/integrations.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/react-task-manager/skills/base44-sdk/references/users.md b/evals/fixtures/react-task-manager/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/react-task-manager/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/fixtures/skill-check/AGENTS.md b/evals/fixtures/skill-check/AGENTS.md new file mode 100644 index 0000000..997c610 --- /dev/null +++ b/evals/fixtures/skill-check/AGENTS.md @@ -0,0 +1,8 @@ +# Skill Check + +A minimal Base44 project for testing skill loading. + +## Available Skills + +- **base44-cli**: CLI for managing Base44 entities, functions, and deployment +- **base44-sdk**: SDK for building features with Base44 APIs diff --git a/evals/fixtures/skill-check/project/.claude/settings.json b/evals/fixtures/skill-check/project/.claude/settings.json new file mode 100644 index 0000000..ab60505 --- /dev/null +++ b/evals/fixtures/skill-check/project/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "permissions": { + "allow": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"] + }, + "skills": ["./skills"] +} diff --git a/evals/fixtures/skill-check/project/CLAUDE.md b/evals/fixtures/skill-check/project/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/fixtures/skill-check/project/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/fixtures/skill-check/project/base44/config.jsonc b/evals/fixtures/skill-check/project/base44/config.jsonc new file mode 100644 index 0000000..268e9ad --- /dev/null +++ b/evals/fixtures/skill-check/project/base44/config.jsonc @@ -0,0 +1,4 @@ +{ + "name": "Skill Check App", + "description": "Simple app for testing skill loading" +} diff --git a/evals/fixtures/skill-check/project/package.json b/evals/fixtures/skill-check/project/package.json new file mode 100644 index 0000000..4bb2ac9 --- /dev/null +++ b/evals/fixtures/skill-check/project/package.json @@ -0,0 +1,4 @@ +{ + "name": "skill-check", + "version": "1.0.0" +} diff --git a/evals/fixtures/skill-check/skills/base44-cli/SKILL.md b/evals/fixtures/skill-check/skills/base44-cli/SKILL.md new file mode 100644 index 0000000..3c33f05 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/SKILL.md @@ -0,0 +1,384 @@ +--- +name: base44-cli +description: "The base44 CLI is used for EVERYTHING related to base44 projects: resource configuration (entities, backend functions, ai agents), initialization and actions (resource creation, deployment). This skill is the place for learning about how to configure resources. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 CLI + +Create and manage Base44 apps (projects) using the Base44 CLI tool. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **NO** (new project scenario): + - This skill (base44-cli) handles the request + - Guide user through project initialization + - Do NOT activate base44-sdk yet +3. If **YES** (existing project scenario): + - Transfer to base44-sdk skill for implementation + - This skill only handles CLI commands (login, deploy, entities push) + +## Critical: Local Installation Only + +NEVER call `base44` directly. The CLI is installed locally as a dev dependency and must be accessed via a package manager: + +- `npx base44 ` (npm - recommended) +- `yarn base44 ` (yarn) +- `pnpm base44 ` (pnpm) + +WRONG: `base44 login` +RIGHT: `npx base44 login` + +## MANDATORY: Authentication Check at Session Start + +**CRITICAL**: At the very start of every AI session when this skill is activated, you MUST: + +1. **Check authentication status** by running: + ```bash + npx base44 whoami + ``` + +2. **If the user is logged in** (command succeeds and shows an email): + - Continue with the requested task + +3. **If the user is NOT logged in** (command fails or shows an error): + - **STOP immediately** + - **DO NOT proceed** with any CLI operations + - **Ask the user to login manually** by running: + ```bash + npx base44 login + ``` + - Wait for the user to confirm they have logged in before continuing + +**This check is mandatory and must happen before executing any other Base44 CLI commands.** + +## Overview + +The Base44 CLI provides command-line tools for authentication, creating projects, managing entities, and deploying Base44 applications. It is framework-agnostic and works with popular frontend frameworks like Vite, Next.js, and Create React App, Svelte, Vue, and more. + +## When to Use This Skill vs base44-sdk + +**Use base44-cli when:** +- Creating a **NEW** Base44 project from scratch +- Initializing a project in an empty directory +- Directory is missing `base44/config.jsonc` +- User mentions: "create a new project", "initialize project", "setup a project", "start a new Base44 app" +- Deploying, pushing entities, or authenticating via CLI +- Working with CLI commands (`npx base44 ...`) + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists +- Writing JavaScript/TypeScript code using Base44 SDK +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code" + +**Skill Dependencies:** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first +- `base44-sdk` assumes a Base44 project is already initialized + +**State Check Logic:** +Before selecting a skill, check: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Project Structure + +A Base44 project combines a standard frontend project with a `base44/` configuration folder: + +``` +my-app/ +├── base44/ # Base44 configuration (created by CLI) +│ ├── config.jsonc # Project settings, site config +│ ├── entities/ # Entity schema definitions +│ │ ├── task.jsonc +│ │ └── board.jsonc +│ ├── functions/ # Backend functions (optional) +│ │ └── my-function/ +│ │ ├── function.jsonc +│ │ └── index.ts +│ └── agents/ # Agent configurations (optional) +│ └── support_agent.jsonc +├── src/ # Frontend source code +│ ├── api/ +│ │ └── base44Client.js # Base44 SDK client +│ ├── pages/ +│ ├── components/ +│ └── main.jsx +├── index.html # SPA entry point +├── package.json +└── vite.config.js # Or your framework's config +``` + +**Key files:** +- `base44/config.jsonc` - Project name, description, site build settings +- `base44/entities/*.jsonc` - Data model schemas (see Entity Schema section) +- `base44/agents/*.jsonc` - Agent configurations (optional) +- `src/api/base44Client.js` - Pre-configured SDK client for frontend use + +**config.jsonc example:** +```jsonc +{ + "name": "My App", // Required: project name + "description": "App description", // Optional: project description + "entitiesDir": "./entities", // Optional: default "entities" + "functionsDir": "./functions", // Optional: default "functions" + "agentsDir": "./agents", // Optional: default "agents" + "site": { // Optional: site deployment config + "installCommand": "npm install", // Optional: install dependencies + "buildCommand": "npm run build", // Optional: build command + "serveCommand": "npm run dev", // Optional: local dev server + "outputDirectory": "./dist" // Optional: build output directory + } +} +``` + +**Config properties:** + +| Property | Description | Default | +|----------|-------------|---------| +| `name` | Project name (required) | - | +| `description` | Project description | - | +| `entitiesDir` | Directory for entity schemas | `"entities"` | +| `functionsDir` | Directory for backend functions | `"functions"` | +| `agentsDir` | Directory for agent configs | `"agents"` | +| `site.installCommand` | Command to install dependencies | - | +| `site.buildCommand` | Command to build the project | - | +| `site.serveCommand` | Command to run dev server | - | +| `site.outputDirectory` | Build output directory for deployment | - | + +## Installation + +Install the Base44 CLI as a dev dependency in your project: + +```bash +npm install --save-dev base44 +``` + +**Important:** Never assume or hardcode the `base44` package version. Always install without a version specifier to get the latest version. + +Then run commands using `npx`: + +```bash +npx base44 +``` + +**Note:** All commands in this documentation use `npx base44`. You can also use `yarn base44`, or `pnpm base44` if preferred. + +## Available Commands + +### Authentication + +| Command | Description | Reference | +| --------------- | ----------------------------------------------- | ------------------------------------------- | +| `base44 login` | Authenticate with Base44 using device code flow | [auth-login.md](references/auth-login.md) | +| `base44 logout` | Logout from current device | [auth-logout.md](references/auth-logout.md) | +| `base44 whoami` | Display current authenticated user | [auth-whoami.md](references/auth-whoami.md) | + +### Project Management + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 create` | Create a new Base44 project from a template | [create.md](references/create.md) ⚠️ **MUST READ** | +| `base44 link` | Link an existing local project to Base44 | [link.md](references/link.md) | +| `base44 dashboard` | Open the app dashboard in your browser | [dashboard.md](references/dashboard.md) | + +### Deployment + +| Command | Description | Reference | +|---------|-------------|-----------| +| `base44 deploy` | Deploy all resources (entities, functions, site) | [deploy.md](references/deploy.md) | + +### Entity Management + +| Action / Command | Description | Reference | +| ---------------------- | ------------------------------------------- | --------------------------------------------------- | +| Create Entities | Define entities in `base44/entities` folder | [entities-create.md](references/entities-create.md) | +| `base44 entities push` | Push local entities to Base44 | [entities-push.md](references/entities-push.md) | + +#### Entity Schema (Quick Reference) + +ALWAYS follow this exact structure when creating entity files: + +**File naming:** `base44/entities/{kebab-case-name}.jsonc` (e.g., `team-member.jsonc` for `TeamMember`) + +**Schema template:** +```jsonc +{ + "name": "EntityName", + "type": "object", + "properties": { + "field_name": { + "type": "string", + "description": "Field description" + } + }, + "required": ["field_name"] +} +``` + +**Field types:** `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` +**String formats:** `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` +**For enums:** Add `"enum": ["value1", "value2"]` and optionally `"default": "value1"` +**Entity names:** Must be alphanumeric only (pattern: `/^[a-zA-Z0-9]+$/`) + +For complete documentation, see [entities-create.md](references/entities-create.md). + +### Function Management + +| Action / Command | Description | Reference | +| ------------------------- | --------------------------------------------- | ------------------------------------------------------- | +| Create Functions | Define functions in `base44/functions` folder | [functions-create.md](references/functions-create.md) | +| `base44 functions deploy` | Deploy local functions to Base44 | [functions-deploy.md](references/functions-deploy.md) | + +### Agent Management + +Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. Use these commands to manage agent configurations. + +| Action / Command | Description | Reference | +| ----------------------- | --------------------------------------- | ----------------------------------------------- | +| Create Agents | Define agents in `base44/agents` folder | See Agent Schema below | +| `base44 agents pull` | Pull remote agents to local files | [agents-pull.md](references/agents-pull.md) | +| `base44 agents push` | Push local agents to Base44 | [agents-push.md](references/agents-push.md) | + +**Note:** Agent commands perform full synchronization - pushing replaces all remote agents with local ones, and pulling replaces all local agents with remote ones. + +#### Agent Schema (Quick Reference) + +**File naming:** `base44/agents/{agent_name}.jsonc` (e.g., `support_agent.jsonc`) + +**Schema template:** +```jsonc +{ + "name": "agent_name", + "description": "Brief description of what this agent does", + "instructions": "Detailed instructions for the agent's behavior", + "tool_configs": [ + // Entity tool - gives agent access to entity operations + { "entity_name": "tasks", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" +} +``` + +**Naming rules:** +- Agent names must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores, 1-100 chars) +- Valid: `support_agent`, `order_bot` +- Invalid: `Support-Agent`, `OrderBot` + +**Required fields:** `name`, `description`, `instructions` +**Optional fields:** `tool_configs` (defaults to `[]`), `whatsapp_greeting` + +**Tool config types:** +- **Entity tools**: `entity_name` + `allowed_operations` (array of: `read`, `create`, `update`, `delete`) +- **Backend function tools**: `function_name` + `description` + +### Site Deployment + +| Command | Description | Reference | +| -------------------- | ----------------------------------------- | ------------------------------------------- | +| `base44 site deploy` | Deploy built site files to Base44 hosting | [site-deploy.md](references/site-deploy.md) | + +**SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## Quick Start + +1. Install the CLI in your project: + ```bash + npm install --save-dev base44 + ``` + +2. Authenticate with Base44: + ```bash + npx base44 login + ``` + +3. Create a new project (ALWAYS provide name and `--path` flag): + ```bash + npx base44 create my-app -p . + ``` + +4. Build and deploy everything: + ```bash + npm run build + npx base44 deploy -y + ``` + +Or deploy individual resources: +- `npx base44 entities push` - Push entities only +- `npx base44 functions deploy` - Deploy functions only +- `npx base44 agents push` - Push agents only +- `npx base44 site deploy -y` - Deploy site only + +## Common Workflows + +### Creating a New Project + +**⚠️ MANDATORY: Before running `base44 create`, you MUST read [create.md](references/create.md) for:** +- **Template selection** - Choose the correct template (`backend-and-client` vs `backend-only`) +- **Correct workflow** - Different templates require different setup steps +- **Common pitfalls** - Avoid folder creation errors that cause failures + +Failure to follow the create.md instructions will result in broken project scaffolding. + +### Linking an Existing Project +```bash +# If you have base44/config.jsonc but no .app.jsonc +npx base44 link --create --name my-app +``` + +### Deploying All Changes +```bash +# Build your project first +npm run build + +# Deploy everything (entities, functions, and site) +npx base44 deploy -y +``` + +### Deploying Individual Resources +```bash +# Push only entities +npx base44 entities push + +# Deploy only functions +npx base44 functions deploy + +# Push only agents +npx base44 agents push + +# Deploy only site +npx base44 site deploy -y +``` + +### Opening the Dashboard +```bash +# Open app dashboard in browser +npx base44 dashboard +``` + +## Authentication + +Most commands require authentication. If you're not logged in, the CLI will automatically prompt you to login. Your session is stored locally and persists across CLI sessions. + +## Troubleshooting + +| Error | Solution | +| --------------------------- | ----------------------------------------------------------------------------------- | +| Not authenticated | Run `npx base44 login` first | +| No entities found | Ensure entities exist in `base44/entities/` directory | +| Entity not recognized | Ensure file uses kebab-case naming (e.g., `team-member.jsonc` not `TeamMember.jsonc`) | +| No functions found | Ensure functions exist in `base44/functions/` with valid `function.jsonc` configs | +| No agents found | Ensure agents exist in `base44/agents/` directory with valid `.jsonc` configs | +| Invalid agent name | Agent names must be lowercase alphanumeric with underscores only | +| No site configuration found | Check that `site.outputDirectory` is configured in project config | +| Site deployment fails | Ensure you ran `npm run build` first and the build succeeded | diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/agents-pull.md b/evals/fixtures/skill-check/skills/base44-cli/references/agents-pull.md new file mode 100644 index 0000000..6a700f0 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/agents-pull.md @@ -0,0 +1,73 @@ +# base44 agents pull + +Pull AI agent configurations from Base44 to local files. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents pull +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Fetches all agents from Base44 +2. Writes agent files to the `base44/agents/` directory +3. Deletes local agent files that don't exist remotely +4. Reports written and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must be linked to a Base44 app + +## Output + +```bash +$ npx base44 agents pull + +Fetching agents from Base44... +✓ Agents fetched successfully + +Writing agent files... +✓ Agent files written successfully + +Written: support_agent, order_bot +Deleted: old_agent + +Pulled 2 agents to base44/agents +``` + +## Agent Synchronization + +The pull operation synchronizes remote agents to your local files: + +- **Written**: Agent files created or updated from remote +- **Deleted**: Local agent files removed (didn't exist remotely) + +**Warning**: This operation replaces all local agent configurations with remote versions. Any local changes not pushed to Base44 will be overwritten. + +## Error Handling + +If no agents exist on Base44: +```bash +$ npx base44 agents pull +No agents found on Base44 +``` + +## Use Cases + +- Sync agent configurations to a new development machine +- Get the latest agent configurations from your team +- Restore local agent files after accidental deletion +- Start working on an existing project with agents + +## Notes + +- This command syncs agent configurations, not conversation data +- Agent files are stored as `.jsonc` in the `base44/agents/` directory +- The directory location is configurable via `agentsDir` in `config.jsonc` +- Use `base44 agents push` to upload local changes to Base44 diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/agents-push.md b/evals/fixtures/skill-check/skills/base44-cli/references/agents-push.md new file mode 100644 index 0000000..fa2795b --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/agents-push.md @@ -0,0 +1,156 @@ +# base44 agents push + +Push local AI agent configurations to Base44. Agents are conversational AI assistants that can interact with users, access your app's entities, and call backend functions. + +## Syntax + +```bash +npx base44 agents push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Reads all agent files from the `base44/agents/` directory +2. Validates agent configurations +3. Displays the count of agents to be pushed +4. Uploads agents to the Base44 backend +5. Reports the results: created, updated, and deleted agents + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have agent definitions in the `base44/agents/` folder + +## Output + +```bash +$ npx base44 agents push + +Found 2 agents to push +Pushing agents to Base44... + +Created: support_agent +Updated: order_bot +Deleted: old_agent + +✓ Agents pushed to Base44 +``` + +## Agent Synchronization + +The push operation synchronizes your local agents with Base44: + +- **Created**: New agents that didn't exist in Base44 +- **Updated**: Existing agents with modified configuration +- **Deleted**: Agents that were removed from your local configuration + +**Warning**: This is a full sync operation. Agents removed locally will be deleted from Base44. + +## Error Handling + +If no agents are found in your project: +```bash +$ npx base44 agents push +No local agents found - this will delete all remote agents +``` + +If an agent has an invalid name: +```bash +$ npx base44 agents push +Error: Agent name must be lowercase alphanumeric with underscores +``` + +## Agent Configuration Schema + +Each agent file should be a `.jsonc` file in `base44/agents/` with this structure: + +```jsonc +{ + "name": "agent_name", // Required: lowercase alphanumeric with underscores, 1-100 chars + "description": "Brief description of what this agent does", // Required: min 1 char + "instructions": "Detailed instructions for the agent's behavior", // Required: min 1 char + "tool_configs": [ // Optional: defaults to [] + // Entity tool - gives agent access to entity operations + { "entity_name": "Task", "allowed_operations": ["read", "create", "update", "delete"] }, + // Backend function tool - gives agent access to a function + { "function_name": "send_email", "description": "Send an email notification" } + ], + "whatsapp_greeting": "Hello! How can I help you today?" // Optional +} +``` + +**Naming rules:** +- **Agent names** must match pattern: `/^[a-z0-9_]+$/` (lowercase alphanumeric with underscores only, 1-100 characters) + - Valid: `support_agent`, `order_bot`, `task_helper` + - Invalid: `Support-Agent`, `OrderBot`, `task helper` +- **Agent file names** must use underscores (matching the agent name) + - Valid: `support_agent.jsonc`, `order_bot.jsonc` + - Invalid: `support-agent.jsonc` (hyphens not allowed) +- **Entity names in `tool_configs`** must use PascalCase (matching the entity's `name` field) + - Valid: `"entity_name": "Task"`, `"entity_name": "TeamMember"` + - Invalid: `"entity_name": "task"`, `"entity_name": "team_member"` + +**Required fields:** +- `name`: Required, must follow naming rules above +- `description`: Required, minimum 1 character +- `instructions`: Required, minimum 1 character +- `tool_configs`: Optional, defaults to empty array +- `whatsapp_greeting`: Optional + +### Common Mistake: Wrong tool_configs Format + +**WRONG** - Do NOT use `tools` with `type` and `entity`: +```jsonc +{ + "name": "my_agent", + "tools": [ // ❌ WRONG + { "type": "entity_query", "entity": "Task" } + ] +} +``` + +**CORRECT** - Use `tool_configs` with `entity_name` and `allowed_operations`: +```jsonc +{ + "name": "my_agent", + "tool_configs": [ // ✅ CORRECT + { "entity_name": "Task", "allowed_operations": ["read"] } + ] +} +``` + +### Best Practices for Agent Instructions + +When giving agents access to entities, be explicit in the instructions about using the tools: + +```jsonc +{ + "name": "support_agent", + "instructions": "You are a helpful support agent.\n\nIMPORTANT: You have access to customer data through entity tools. When users ask about their orders or account:\n1. ALWAYS use the Order entity tool to query their order history\n2. Use the Customer entity tool to look up account details\n3. Analyze the data and provide personalized responses\n\nAlways query the relevant entities first before answering questions about user data.", + "tool_configs": [ + { "entity_name": "Order", "allowed_operations": ["read"] }, + { "entity_name": "Customer", "allowed_operations": ["read"] } + ] +} +``` + +Without explicit instructions to use the entity tools, the agent may not proactively query user data when asked. + +## Use Cases + +- After defining new agents in your project +- When modifying existing agent configurations +- To sync agent changes before testing +- As part of your development workflow when agent behavior changes + +## Notes + +- This command syncs the agent configuration, not conversation data +- Changes are applied to your Base44 project immediately +- Make sure to test agent changes in a development environment first +- Agent definitions are located in the `base44/agents/` directory +- Use `base44 agents pull` to download agents from Base44 diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/auth-login.md b/evals/fixtures/skill-check/skills/base44-cli/references/auth-login.md new file mode 100644 index 0000000..adebd27 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/auth-login.md @@ -0,0 +1,54 @@ +# base44 login + +Authenticate with Base44 using device code flow. + +## Syntax + +```bash +npx base44 login +``` + +## Authentication + +**Required**: No (this is the login command itself) + +## How It Works + +The login command uses OAuth 2.0 device code flow for authentication: + +1. Generates a device code for authentication +2. Displays a verification code and verification URI +3. Directs you to visit the URI and enter the code +4. Polls for authentication completion (up to device code expiration) +5. Retrieves access and refresh tokens upon successful authentication +6. Fetches and displays your user information +7. Saves authentication data locally with expiration timestamp + +## Interactive Flow + +```bash +$ npx base44 login + +Please visit: https://auth.base44.com/device +Enter code: ABCD-EFGH + +Waiting for authentication... +✓ Successfully authenticated! + +Logged in as: user@example.com +``` + +## Session Management + +- Authentication tokens are stored locally on your device +- Tokens include expiration timestamps +- The session persists across CLI sessions +- Other commands will automatically use your stored credentials +- Use `npx base44 logout` to clear your session +- Use `npx base44 whoami` to check your current authentication status + +## Notes + +- You only need to login once per device +- If your session expires, you'll be prompted to login again when running authenticated commands +- The CLI automatically prompts for login when you run commands that require authentication diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/auth-logout.md b/evals/fixtures/skill-check/skills/base44-cli/references/auth-logout.md new file mode 100644 index 0000000..33c2ddf --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/auth-logout.md @@ -0,0 +1,32 @@ +# base44 logout + +Logout from current device and clear stored authentication data. + +## Syntax + +```bash +npx base44 logout +``` + +## Authentication + +**Required**: No + +## What It Does + +- Deletes stored authentication data from your device +- Clears your local session +- Removes access and refresh tokens + +## Output + +```bash +$ npx base44 logout +Logged out successfully +``` + +## Notes + +- You can logout even if you're not currently logged in (no error) +- After logout, you'll need to run `npx base44 login` again to use authenticated commands +- This only affects the current device; your Base44 account remains active diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/auth-whoami.md b/evals/fixtures/skill-check/skills/base44-cli/references/auth-whoami.md new file mode 100644 index 0000000..abe450c --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/auth-whoami.md @@ -0,0 +1,37 @@ +# base44 whoami + +Display the currently authenticated user. + +## Syntax + +```bash +npx base44 whoami +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +- Reads stored authentication data +- Displays the email of the currently logged-in user + +## Output + +```bash +$ npx base44 whoami +Logged in as: user@example.com +``` + +## Use Cases + +- Verify you're logged in before running other commands +- Check which account you're currently using +- Confirm authentication is working properly +- Useful in scripts or automation to verify credentials + +## Notes + +- If you're not logged in, the command will prompt you to authenticate first +- The email displayed matches your Base44 account email diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/create.md b/evals/fixtures/skill-check/skills/base44-cli/references/create.md new file mode 100644 index 0000000..7580645 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/create.md @@ -0,0 +1,111 @@ +# base44 create + +Creates a new Base44 project from a template. This command is framework-agnostic and can either scaffold a complete project or add Base44 configuration to an existing project. + +## Critical: Non-Interactive Mode Required + +ALWAYS provide both the project name AND `--path` flag. Without both, the command opens an interactive TUI which agents cannot use properly. + +WRONG: `npx base44 create` +WRONG: `npx base44 create my-app` +RIGHT: `npx base44 create my-app -p ./my-app` + +## Syntax + +```bash +npx base44 create [name] --path [options] +``` + +## Arguments & Options + +| Argument/Option | Description | Required | +|--------|-------------|----------| +| `name` | Project name (positional argument) | Yes* | +| `-p, --path ` | Path where to create the project | Yes* | +| `-t, --template ` | Template ID (see templates below) | No | +| `--deploy` | Build and deploy the site (includes pushing entities) | No | +| `--no-skills` | Skip AI agent skills installation (skills are added by default) | No | + +*Required for non-interactive mode. Both `name` and `--path` must be provided together. + +## Template Selection (CRITICAL - Choose Appropriately) + +**You MUST select the most appropriate template based on user requirements:** + +| Template ID | When to Use | Example Scenarios | +|-------------|-------------|-------------------| +| `backend-and-client` | Creating a NEW full-stack web app from scratch | "Create a task app", "Build me a dashboard", "Make a SaaS app" | +| `backend-only` | Adding Base44 to an EXISTING project OR using a different framework (Next.js, Vue, Svelte, etc.) | "Add Base44 to my project", "I want to use Next.js", "I already have a frontend" | + +**Default Choice:** When the user asks to "create an app" or "build a project" without specifying a particular framework, use `backend-and-client` to provide a complete, production-ready application with Vite + React + Tailwind. + +## The `--path` Flag + +- **For `backend-and-client` template (new projects):** Use a new subfolder path + ```bash + npx base44 create my-app -p ./my-app -t backend-and-client + ``` +- **For `backend-only` template (existing projects):** Use `-p .` in the current directory + ```bash + npx base44 create my-app -p . + ``` + +## Workflow: Using `backend-only` with External Frameworks + +**CRITICAL: The project folder MUST exist BEFORE running `base44 create` with `backend-only`** + +The `backend-only` template only adds Base44 configuration files - it does NOT create a frontend. If you need a frontend with a specific framework: + +```bash +# Step 1: Initialize the frontend project FIRST +npm create vite@latest my-app -- --template react # or vue, svelte, etc. +# OR: npx create-next-app@latest my-app +# OR: any other framework's init command + +# Step 2: Navigate into the created folder +cd my-app + +# Step 3: Install Base44 CLI +npm install --save-dev base44 + +# Step 4: Add Base44 configuration +npx base44 create my-app -p . +``` + +**WARNING:** Do NOT: +- Create an empty folder manually, then try to run `npx create vite` inside it (will fail - folder exists) +- Run `base44 create` with `backend-only` expecting it to create a frontend (it won't) + +**DO:** +- Run the external framework's init command FIRST (it creates its own folder) +- Then run `base44 create` inside that folder with `-p .` + +## Examples + +```bash +# RECOMMENDED: Create full-stack project (for new apps) +npx base44 create my-app -p ./my-app -t backend-and-client + +# Create full-stack and deploy in one step +npx base44 create my-app -p ./my-app -t backend-and-client --deploy + +# Add Base44 to EXISTING project (must be inside the project folder) +npx base44 create my-app -p . + +# Add Base44 to existing project and deploy +npx base44 create my-app -p . --deploy + +# Create without adding AI agent skills +npx base44 create my-app -p . --no-skills +``` + +## What It Does + +1. Applies the selected template to the target path +2. Creates a `base44/` folder with configuration files +3. Registers the project with Base44 backend +4. Creates `base44/.app.jsonc` with the app ID +5. If `--deploy` is used: + - Pushes any entities defined in `base44/entities/` + - Runs install and build commands (for templates with frontend) + - Deploys the site to Base44 hosting diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/dashboard.md b/evals/fixtures/skill-check/skills/base44-cli/references/dashboard.md new file mode 100644 index 0000000..95a534a --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/dashboard.md @@ -0,0 +1,35 @@ +# base44 dashboard + +Opens the Base44 app dashboard in your default web browser. + +## Syntax + +```bash +npx base44 dashboard +``` + +## Options + +This command has no options. + +## What It Does + +1. Reads the project's app ID from `base44/.app.jsonc` +2. Opens the dashboard URL in your default browser + +## Example + +```bash +# Open dashboard for current project +npx base44 dashboard +``` + +## Requirements + +- Must be run from a linked Base44 project directory (contains `base44/.app.jsonc`) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- The dashboard provides a web interface to manage your app's entities, functions, users, and settings +- If you're not in a project directory, the command will fail with an error diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/deploy.md b/evals/fixtures/skill-check/skills/base44-cli/references/deploy.md new file mode 100644 index 0000000..500526f --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/deploy.md @@ -0,0 +1,86 @@ +# base44 deploy + +Deploys all project resources (entities, functions, and site) to Base44 in a single command. + +## Syntax + +```bash +npx base44 deploy [options] +``` + +## Options + +| Option | Description | +|--------|-------------| +| `-y, --yes` | Skip confirmation prompt | + +## What It Deploys + +The command automatically detects and deploys: + +1. **Entities** - All `.jsonc` files in `base44/entities/` +2. **Functions** - All functions in `base44/functions/` +3. **Agents** - All agent configurations in `base44/agents/` +4. **Site** - Built files from `site.outputDirectory` (if configured) + +## Examples + +```bash +# Interactive mode - shows what will be deployed and asks for confirmation +npx base44 deploy + +# Non-interactive - skip confirmation (for CI/CD or agent use) +npx base44 deploy -y +``` + +## Typical Workflow + +```bash +# 1. Make your changes (entities, functions, frontend code) + +# 2. Build the frontend (if you have one) +npm run build + +# 3. Deploy everything +npx base44 deploy -y +``` + +## What It Does + +1. Reads project configuration from `base44/config.jsonc` +2. Detects available resources (entities, functions, site) +3. Shows a summary of what will be deployed +4. Asks for confirmation (unless `-y` flag is used) +5. Deploys all resources in sequence: + - Pushes entity schemas + - Deploys functions + - Pushes agent configurations + - Uploads site files +6. Displays the dashboard URL and app URL (if site was deployed) + +## Requirements + +- Must be run from a linked Base44 project directory +- Must be authenticated (run `npx base44 login` first) +- For site deployment, must run `npm run build` first + +## Output + +After successful deployment: +- **Dashboard**: Link to your app's management dashboard +- **App URL**: Your deployed site's public URL (if site was included) + +## Notes + +- If no resources are found, the command exits with a message +- Use individual commands (`entities push`, `functions deploy`, `site deploy`) if you only want to deploy specific resources +- The site must be built before deployment - this command does not run `npm run build` for you + +## Related Commands + +| Command | Description | +|---------|-------------| +| `base44 entities push` | Push only entities | +| `base44 functions deploy` | Deploy only functions | +| `base44 agents push` | Push only agents | +| `base44 site deploy` | Deploy only the site | diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/entities-create.md b/evals/fixtures/skill-check/skills/base44-cli/references/entities-create.md new file mode 100644 index 0000000..29b40aa --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/entities-create.md @@ -0,0 +1,552 @@ +# Creating Entities + +Base44 entities are defined locally in your project and then pushed to the Base44 backend. + +## Critical: File Naming + +Entity files MUST use kebab-case naming: `{kebab-case-name}.jsonc` + +| Entity Name | File Name | +|-------------|-----------| +| `Task` | `task.jsonc` | +| `TeamMember` | `team-member.jsonc` | +| `ActivityLog` | `activity-log.jsonc` | + +WRONG: `TeamMember.jsonc`, `teamMember.jsonc` +RIGHT: `team-member.jsonc` + +## Table of Contents + +- [Creating Entities](#creating-entities) + - [Entity Directory](#entity-directory) + - [How to Create an Entity](#how-to-create-an-entity) + - [Entity Schema Structure](#entity-schema-structure) + - [Supported Field Types](#supported-field-types) + - [Field Properties](#field-properties) + - [Complete Example](#complete-example) + - [Naming Conventions](#naming-conventions) + - [Relationships Between Entities](#relationships-between-entities) + - [Row Level Security (RLS)](#row-level-security-rls) + - [Field Level Security (FLS)](#field-level-security-fls) + - [Pushing Entities](#pushing-entities) + +## Entity Directory + +All entity definitions must be placed in the `base44/entities/` folder in your project root. Each entity is defined in its own `.jsonc` file. + +Example structure: +``` +my-app/ + base44/ + entities/ + user.jsonc + product.jsonc + order.jsonc +``` + +## How to Create an Entity + +1. Create a new `.jsonc` file in the `base44/entities/` directory +2. Define your entity schema following the structure below +3. Push the changes to Base44 using the CLI + +## Entity Schema Structure + +Each entity file follows a JSON Schema-like structure: + +```jsonc +{ + "name": "EntityName", // PascalCase entity name + "type": "object", // Always "object" + "properties": { + // Define your fields here + }, + "required": ["field1"] // Array of required field names +} +``` + +### Common Mistake: Nested Schema Property + +**WRONG** - Do NOT wrap properties in a `schema` object: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "schema": { // ❌ WRONG - don't use nested "schema" + "type": "object", + "properties": { ... } + } +} +``` + +**CORRECT** - Put `type` and `properties` at the top level: +```jsonc +{ + "name": "Task", + "description": "A task entity", + "type": "object", // ✅ CORRECT - top level + "properties": { ... } // ✅ CORRECT - top level +} +``` + +This is a common mistake that will cause "Invalid schema: Schema must have a 'type' field" errors when pushing entities. + +## Supported Field Types + +### String + +Basic text field: +```jsonc +{ + "title": { + "type": "string", + "description": "Task title" + } +} +``` + +With format: +```jsonc +{ + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + } +} +``` + +Available formats: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` + +### String with Enum + +Constrained to specific values: +```jsonc +{ + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status" + } +} +``` + +### Number + +```jsonc +{ + "position": { + "type": "number", + "description": "Position for ordering" + } +} +``` + +### Integer + +For whole numbers only: +```jsonc +{ + "quantity": { + "type": "integer", + "description": "Item quantity", + "minimum": 0, + "maximum": 1000 + } +} +``` + +### Binary + +For file/blob data: +```jsonc +{ + "attachment": { + "type": "binary", + "description": "File attachment" + } +} +``` + +### Boolean + +```jsonc +{ + "notify_on_change": { + "type": "boolean", + "default": true, + "description": "Enable notifications" + } +} +``` + +### Array of Strings + +```jsonc +{ + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } +} +``` + +### Array of Objects + +```jsonc +{ + "attachments": { + "type": "array", + "description": "File attachments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "url": { "type": "string" }, + "type": { "type": "string" } + } + } + } +} +``` + +## Field Properties + +| Property | Description | +| ------------- | ---------------------------------------------------------------------------------------- | +| `type` | Data type: `string`, `number`, `integer`, `boolean`, `array`, `object`, `binary` | +| `description` | Human-readable description of the field | +| `enum` | Array of allowed values (for strings) | +| `enumNames` | Human-readable labels for enum values (same order as `enum`) | +| `default` | Default value when not provided | +| `format` | Format hint: `date`, `date-time`, `time`, `email`, `uri`, `hostname`, `ipv4`, `ipv6`, `uuid`, `file`, `regex`, `richtext` | +| `items` | Schema for array items | +| `properties` | Nested properties for object types | +| `$ref` | Reference to another schema definition | +| `minLength` | Minimum string length | +| `maxLength` | Maximum string length | +| `pattern` | Regex pattern for string validation | +| `minimum` | Minimum value for numbers | +| `maximum` | Maximum value for numbers | +| `rls` | Field-level security rules (see Field Level Security section) | + +## Complete Example + +Here's a complete entity definition for a Task: + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "description": { + "type": "string", + "description": "Task description" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo", + "description": "Current status of the task" + }, + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "assignee_email": { + "type": "string", + "description": "Email of assigned user" + }, + "priority": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium", + "description": "Task priority" + }, + "due_date": { + "type": "string", + "format": "date", + "description": "Due date" + }, + "labels": { + "type": "array", + "items": { "type": "string" }, + "description": "Task labels/tags" + } + }, + "required": ["title"] +} +``` + +## Naming Conventions + +- **Entity name**: Use PascalCase with alphanumeric characters only (e.g., `Task`, `TeamMember`, `ActivityLog`) + - Must match pattern: `/^[a-zA-Z0-9]+$/` + - Valid: `Task`, `TeamMember`, `Order123` + - Invalid: `Team_Member`, `Team-Member`, `Team Member` +- **File name**: Use kebab-case matching the entity (e.g., `task.jsonc`, `team-member.jsonc`, `activity-log.jsonc`) +- **Field names**: Use snake_case (e.g., `board_id`, `user_email`, `due_date`) + +## Relationships Between Entities + +To create relationships between entities, use ID reference fields: + +```jsonc +{ + "board_id": { + "type": "string", + "description": "Board this task belongs to" + }, + "team_id": { + "type": "string", + "description": "Associated team ID" + } +} +``` + +## Row Level Security (RLS) + +Row Level Security (RLS) controls which records users can access based on their identity and attributes. RLS rules are defined per entity inside the `rls` field of the schema. + +**Important:** If no RLS is defined, all records are accessible to all users. + +### RLS Operations + +RLS supports five operations: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can add new records | +| `read` | Control who can view records | +| `update` | Control who can modify records | +| `delete` | Control who can remove records | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### Permission Values + +Each operation accepts one of the following values: + +1. **`true`** - Allow all users (including anonymous/unauthenticated) +2. **`false`** - Block all users +3. **Condition object** - Allow users matching the condition + +### Template Variables + +Use template variables to reference the current user's attributes: + +| Template | Description | +|----------|-------------| +| `{{user.id}}` | The user's ID | +| `{{user.email}}` | The user's email | +| `{{user.role}}` | The user's role | +| `{{user.data.field_name}}` | Custom field from the user's `data` object | + +### Built-in Entity Attributes + +Every entity record has these built-in attributes available for RLS rules: + +| Attribute | Description | +|-----------|-------------| +| `id` | Unique record identifier | +| `created_date` | Timestamp when record was created | +| `updated_date` | Timestamp when record was last updated | +| `created_by` | Email of the user who created the record | + +### Rule Types + +There are two condition types you can use: + +**1. Entity-to-user comparison** - Compare record fields to the current user's values: +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**2. User condition check** - Check user properties directly using `user_condition`: +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Important limitations:** +- `user_condition` only supports **simple equality** (e.g., `{ "role": "admin" }`) +- For `data.*` field comparisons, you can use operators: `$in`, `$nin`, `$ne`, `$all` +- Logical operators `$or`, `$and`, `$nor` are available for combining conditions +- You cannot filter by entity field values directly (e.g., `{"status": "published"}`) +- Only user-related conditions are allowed + +### RLS Examples + +**Owner-only access:** +```jsonc +{ + "created_by": "{{user.email}}" +} +``` + +**Department-based access:** +```jsonc +{ + "data.department": "{{user.data.department}}" +} +``` + +**Admin-only access:** +```jsonc +{ + "user_condition": { "role": "admin" } +} +``` + +**Complete RLS configuration:** +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "Task title" + }, + "status": { + "type": "string", + "enum": ["todo", "in_progress", "done"], + "default": "todo" + } + }, + "required": ["title"], + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Common RLS Patterns + +**Public create, admin-only management (e.g., contact forms, waitlists):** +```jsonc +{ + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Owner-only access:** +```jsonc +{ + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +**Logged-in users only:** +```jsonc +{ + "rls": { + "create": { "user_condition": { "id": "{{user.id}}" } }, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Limitations + +- **user_condition is equality only:** `user_condition` only supports exact match (e.g., `{ "role": "admin" }`) - no operators +- **No comparison operators on user_condition:** `$gt`, `$lt`, `$regex`, `$expr`, `$where` are NOT supported for user conditions +- **No entity field filtering:** Cannot filter by entity field values (e.g., `{"status": "published"}`) +- **No deeply nested templates:** Templates like `{{user.data.profile.department}}` may not work + +**Supported operators:** +- **Logical operators:** `$or`, `$and`, `$nor` for combining multiple conditions +- **Field operators (for `data.*` fields only):** `$in`, `$nin`, `$ne`, `$all` + +### Complex Access Patterns + +For complex access patterns that require multiple conditions (e.g., "owner OR admin"), you have two options: + +1. **Use the Base44 Dashboard UI** - The dashboard allows adding multiple rules per operation with OR logic +2. **Use separate entities** - Split data into multiple entities with different access rules +3. **Use backend functions** - Implement custom access logic in backend functions + +## Field Level Security (FLS) + +Field Level Security allows you to control access to individual fields within an entity. FLS rules are defined within each field's schema using the `rls` property. + +### FLS Operations + +FLS supports the same operations as entity-level RLS: + +| Operation | Description | +|-----------|-------------| +| `create` | Control who can set this field when creating records | +| `read` | Control who can view this field | +| `update` | Control who can modify this field | +| `delete` | Control who can clear this field | +| `write` | Shorthand for `create`, `update`, and `delete` combined | + +### FLS Example + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Employee name" + }, + "salary": { + "type": "number", + "description": "Employee salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "update": { "user_condition": { "role": "hr" } } + } + }, + "department": { + "type": "string", + "description": "Department name" + } + }, + "required": ["name"] +} +``` + +In this example, only users with the `hr` role can read or update the `salary` field. All users with access to the entity can read/update other fields. + +### FLS Notes + +- If no field-level RLS is defined, the field inherits the entity-level RLS rules +- FLS rules follow the same condition format as entity-level RLS +- Use FLS for sensitive fields like salary, SSN, or internal notes + +## Pushing Entities + +The `entities push` command will push all entities that exist in the `base44/entities` folder. + +```bash +npx base44 entities push +``` + +For more details on the push command, see [entities-push.md](entities-push.md). diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/entities-push.md b/evals/fixtures/skill-check/skills/base44-cli/references/entities-push.md new file mode 100644 index 0000000..63b92d7 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/entities-push.md @@ -0,0 +1,71 @@ +# base44 entities push + +Push local entity definitions to Base44. + +## Syntax + +```bash +npx base44 entities push +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Pushes all entities that exist in the `base44/entities` folder +2. Validates that entities exist in the folder +3. Displays the count of entities to be pushed +4. Uploads entities to the Base44 backend +5. Reports the results: created, updated, and deleted entities + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have entity definitions in the `base44/entities` folder + +## Output + +```bash +$ npx base44 entities push + +Found 3 entities to push +Pushing entities to Base44... + +Created: User, Post +Updated: Comment +Deleted: OldEntity + +✓ Entities pushed successfully +``` + +## Entity Synchronization + +The push operation synchronizes your local entity schema with Base44: + +- **Created**: New entities that didn't exist in Base44 +- **Updated**: Existing entities with modified schema or configuration +- **Deleted**: Entities that were removed from your local configuration + +## Error Handling + +If no entities are found in your project: +```bash +$ npx base44 entities push +No entities found in project +``` + +## Use Cases + +- After defining new entities in your project +- When modifying existing entity schemas +- To sync entity changes before deploying +- As part of your development workflow when data models change + +## Notes + +- This command syncs the entity schema/structure, not the actual data +- Changes are applied to your Base44 project immediately +- Make sure to test entity changes in a development environment first +- Entity definitions are located in the `base44/entities/` directory diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/functions-create.md b/evals/fixtures/skill-check/skills/base44-cli/references/functions-create.md new file mode 100644 index 0000000..c523cdc --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/functions-create.md @@ -0,0 +1,229 @@ +# Creating Functions + +Base44 functions are serverless backend functions that run on Deno. They are defined locally in your project and deployed to the Base44 backend. + +## Function Directory + +All function definitions must be placed in the `base44/functions/` folder in your project. Each function lives in its own subdirectory with a configuration file and entry point. + +Example structure: +``` +my-app/ + base44/ + functions/ + process-order/ + function.jsonc + index.ts + send-notification/ + function.jsonc + index.ts +``` + +## How to Create a Function + +1. Create a new directory in `base44/functions/` with your function name (use kebab-case) +2. Create a `function.jsonc` configuration file in the directory +3. Create the entry point file (e.g., `index.ts`) +4. Deploy the function using the CLI + +## Function Configuration + +Each function requires a `function.jsonc` configuration file: + +```jsonc +{ + "name": "my-function", + "entry": "index.ts" +} +``` + +### Configuration Properties + +| Property | Description | Required | +|----------|-------------|----------| +| `name` | Function name (must match `/^[^.]+$/` - no dots allowed) | Yes | +| `entry` | Entry point file path relative to the function directory (min 1 char) | Yes | + +## Entry Point File + +Functions run on Deno and must export using `Deno.serve()`. Use `npm:` prefix for npm packages. + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### Request Object + +The function receives a standard Deno `Request` object: +- `req.json()` - Parse JSON body +- `req.text()` - Get raw text body +- `req.headers` - Access request headers +- `req.method` - HTTP method + +### Response Object + +Return using `Response.json()` for JSON responses: + +```typescript +// Success response +return Response.json({ data: result }); + +// Error response with status code +return Response.json({ error: "Something went wrong" }, { status: 400 }); + +// Not found +return Response.json({ error: "Order not found" }, { status: 404 }); +``` + +## Complete Example + +### Directory Structure +``` +base44/ + functions/ + process-order/ + function.jsonc + index.ts +``` + +### function.jsonc +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +### index.ts +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + // Validate input + if (!orderId) { + return Response.json( + { error: "Order ID is required" }, + { status: 400 } + ); + } + + // Fetch and process the order + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ + success: true, + orderId: order.id, + processedAt: new Date().toISOString() + }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Using Service Role Access + +For admin-level operations, use `asServiceRole`: + +```typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +## Using Secrets + +Access environment variables configured in the app dashboard: + +```typescript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +## Naming Conventions + +- **Directory name**: Use kebab-case (e.g., `process-order`, `send-notification`) +- **Function name**: Match the directory name, must match pattern `/^[^.]+$/` (no dots allowed) + - Valid: `process-order`, `send_notification`, `myFunction` + - Invalid: `process.order`, `send.notification.v2` +- **Entry file**: Typically `index.ts` or `index.js` + +## Deploying Functions + +After creating your function, deploy it to Base44: + +```bash +npx base44 functions deploy +``` + +For more details on deploying, see [functions-deploy.md](functions-deploy.md). + +## Notes + +- Functions run on Deno runtime, not Node.js +- Use `npm:` prefix for npm packages (e.g., `npm:@base44/sdk`) +- Use `createClientFromRequest(req)` to get a client that inherits the caller's auth context +- Configure secrets via app dashboard for API keys +- Make sure to handle errors gracefully and return appropriate HTTP status codes + +## Common Mistakes + +| Wrong | Correct | Why | +|-------|---------|-----| +| `functions/myFunction.js` (single file) | `functions/my-function/index.ts` + `function.jsonc` | Functions require subdirectory with config | +| `import { ... } from "@base44/sdk"` | `import { ... } from "npm:@base44/sdk"` | Deno requires `npm:` prefix for npm packages | +| `MyFunction` or `myFunction` directory | `my-function` directory | Use kebab-case for directory names | diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/functions-deploy.md b/evals/fixtures/skill-check/skills/base44-cli/references/functions-deploy.md new file mode 100644 index 0000000..f89d796 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/functions-deploy.md @@ -0,0 +1,78 @@ +# base44 functions deploy + +Deploy local function definitions to Base44. + +## Syntax + +```bash +npx base44 functions deploy +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## What It Does + +1. Scans the `base44/functions/` directory for function definitions +2. Validates that functions exist and have valid configurations +3. Displays the count of functions to be deployed +4. Uploads function code and configuration to Base44 +5. Reports the results: deployed and deleted functions + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have function definitions in the `base44/functions/` folder +- Each function must have a valid `function.jsonc` config file + +## Output + +```bash +$ npx base44 functions deploy + +Found 2 functions to deploy +Deploying functions to Base44... + +Deployed: process-order, send-notification +Deleted: old-function + +✓ Functions deployed successfully +``` + +## Function Synchronization + +The deploy operation synchronizes your local functions with Base44: + +- **Deployed**: Functions that were created or updated +- **Deleted**: Functions that were removed from your local configuration + +## Error Handling + +If no functions are found in your project: +```bash +$ npx base44 functions deploy +No functions found. Create functions in the 'functions' directory. +``` + +If a function has configuration errors: +```bash +$ npx base44 functions deploy +Function deployment errors: +'my-function' function: Entry point cannot be empty +``` + +## Use Cases + +- After creating new functions in your project +- When modifying existing function code or configuration +- To sync function changes before testing +- As part of your development workflow when backend logic changes + +## Notes + +- This command deploys the function code and configuration +- Changes are applied to your Base44 project immediately +- Make sure to test functions in a development environment first +- Function definitions are located in the `base44/functions/` directory +- For how to create functions, see [functions-create.md](functions-create.md) diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/link.md b/evals/fixtures/skill-check/skills/base44-cli/references/link.md new file mode 100644 index 0000000..0b009ce --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/link.md @@ -0,0 +1,81 @@ +# base44 link + +Links an existing local Base44 project to a Base44 app in the cloud. Use this when you have a `base44/config.jsonc` but haven't connected it to a Base44 app yet. + +## Critical: When to Use Link vs Create + +| Scenario | Command | +|----------|---------| +| Starting fresh, no `base44/` folder | `npx base44 create` | +| Have `base44/config.jsonc` but no `.app.jsonc` | `npx base44 link` | +| Project already linked (has `.app.jsonc`) | Already done, use `deploy` | + +## Syntax + +```bash +npx base44 link [options] +``` + +## Options + +| Option | Description | Required | +|--------|-------------|----------| +| `-c, --create` | Create a new project (skip selection prompt) | No | +| `-n, --name ` | Project name (required when `--create` is used) | With `--create` | +| `-d, --description ` | Project description | No | +| `-p, --projectId ` | Project ID to link to an existing project (skip selection prompt) | No | + +## Non-Interactive Mode + +For CI/CD or agent use: + +**Create a new project:** +```bash +npx base44 link --create --name my-app +``` + +**Link to an existing project:** +```bash +npx base44 link --projectId +``` + +WRONG: `npx base44 link --create` (missing --name) +WRONG: `npx base44 link --create --projectId ` (cannot use both) +RIGHT: `npx base44 link --create --name my-app` +RIGHT: `npx base44 link --projectId ` + +## Examples + +```bash +# Interactive mode - prompts for project details +npx base44 link + +# Non-interactive - create and link in one step +npx base44 link --create --name my-app + +# With description +npx base44 link --create --name my-app --description "My awesome app" + +# Link to a specific existing project by ID +npx base44 link --projectId abc123 +``` + +## What It Does + +1. Finds the `base44/config.jsonc` in the current directory (or parent directories) +2. Verifies no `.app.jsonc` exists (project not already linked) +3. Either: + - Creates a new Base44 app in the cloud (with `--create`), OR + - Links to an existing project (with `--projectId` or interactive selection) +4. Writes the app ID to `base44/.app.jsonc` + +## Requirements + +- Must have `base44/config.jsonc` in the project +- Must NOT have `base44/.app.jsonc` (use `deploy` if already linked) +- Must be authenticated (run `npx base44 login` first) + +## Notes + +- After linking, you can deploy resources with `npx base44 deploy` +- The `.app.jsonc` file should be git-ignored (contains your app ID) diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/rls-examples.md b/evals/fixtures/skill-check/skills/base44-cli/references/rls-examples.md new file mode 100644 index 0000000..7d3ef42 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/rls-examples.md @@ -0,0 +1,463 @@ +# RLS Examples + +Practical Row-Level Security patterns for common application types. + +**Important:** Base44 RLS supports: +- **Logical operators:** `$or`, `$and`, `$nor` for combining conditions +- **Field operators (for `data.*` fields):** `$in`, `$nin`, `$ne`, `$all` +- **user_condition:** Equality only (no operators) + +## Contents +- [Simple Patterns (JSON Schema)](#simple-patterns-json-schema) +- [Using Operators](#using-operators) +- [Field-Level Security Examples](#field-level-security-examples) +- [Complex Patterns (Dashboard UI or Backend)](#complex-patterns-dashboard-ui-or-backend) +- [Best Practices](#best-practices) + +--- + +## Simple Patterns (JSON Schema) + +These patterns work with the JSON schema RLS format. + +### Todo App - Owner-only access + +Users see and manage only their own tasks. + +```jsonc +{ + "name": "Task", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "completed": { "type": "boolean" }, + "priority": { "type": "string", "enum": ["low", "medium", "high"] }, + "due_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Contact Form - Public create, admin-only read + +Anyone can submit, only admins can view submissions. + +```jsonc +{ + "name": "ContactSubmission", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "message": { "type": "string" } + }, + "rls": { + "create": true, + "read": { "user_condition": { "role": "admin" } }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### User Profile - Self-management + +Users can only access their own profile. + +```jsonc +{ + "name": "UserProfile", + "type": "object", + "properties": { + "name": { "type": "string" }, + "avatar_url": { "type": "string" }, + "bio": { "type": "string" }, + "preferences": { "type": "object" } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Department Data - Same department access + +Users can only see records from their department. + +```jsonc +{ + "name": "DepartmentAnnouncement", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "department": { "type": "string" } + }, + "rls": { + "create": { "user_condition": { "role": "manager" } }, + "read": { "data.department": "{{user.data.department}}" }, + "update": { "user_condition": { "role": "manager" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +### Subscription - Admin-managed, user-readable via email field + +```jsonc +{ + "name": "Subscription", + "type": "object", + "properties": { + "user_email": { "type": "string" }, + "tier": { "type": "string", "enum": ["free", "basic", "pro", "enterprise"] }, + "credits": { "type": "number" }, + "renewal_date": { "type": "string", "format": "date" } + }, + "rls": { + "create": { "user_condition": { "role": "admin" } }, + "read": { "data.user_email": "{{user.email}}" }, + "update": { "user_condition": { "role": "admin" } }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Note:** This pattern only allows users to read their own subscription. Admins need to use the Dashboard UI to configure additional read access for themselves. + +### Private Data - Owner-only + +```jsonc +{ + "name": "PrivateNotes", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } } + }, + "rls": { + "create": true, + "read": { "created_by": "{{user.email}}" }, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +### Public Read, Authenticated Write + +Anyone can read, only logged-in users can create/edit their own records. + +```jsonc +{ + "name": "BlogPost", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" }, + "author_email": { "type": "string" } + }, + "rls": { + "create": true, + "read": true, + "update": { "created_by": "{{user.email}}" }, + "delete": { "created_by": "{{user.email}}" } + } +} +``` + +--- + +## Using Operators + +### Logical Operators + +Combine multiple conditions using `$or`, `$and`, or `$nor`: + +**Owner OR Admin access:** +```jsonc +{ + "name": "Document", + "type": "object", + "properties": { + "title": { "type": "string" }, + "content": { "type": "string" } + }, + "rls": { + "create": true, + "read": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "update": { + "$or": [ + { "created_by": "{{user.email}}" }, + { "user_condition": { "role": "admin" } } + ] + }, + "delete": { "user_condition": { "role": "admin" } } + } +} +``` + +**Multiple roles with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "admin" } }, + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + } + } +} +``` + +### Field Operators for data.* Fields + +Use `$in`, `$nin`, `$ne`, `$all` for comparing entity data fields: + +**Access based on tags ($in):** +```jsonc +{ + "rls": { + "read": { + "data.category": { "$in": ["public", "shared"] } + } + } +} +``` + +**Exclude specific statuses ($nin):** +```jsonc +{ + "rls": { + "read": { + "data.status": { "$nin": ["archived", "deleted"] } + } + } +} +``` + +**Not equal ($ne):** +```jsonc +{ + "rls": { + "read": { + "data.visibility": { "$ne": "private" } + } + } +} +``` + +**All tags must match ($all):** +```jsonc +{ + "rls": { + "read": { + "data.required_tags": { "$all": ["approved", "reviewed"] } + } + } +} +``` + +### Combining Logical and Field Operators + +```jsonc +{ + "rls": { + "read": { + "$and": [ + { "data.status": { "$ne": "draft" } }, + { + "$or": [ + { "created_by": "{{user.email}}" }, + { "data.visibility": "public" } + ] + } + ] + } + } +} +``` + +--- + +## Field-Level Security Examples + +Control access to specific fields within an entity. + +### Sensitive Salary Field + +```jsonc +{ + "name": "Employee", + "type": "object", + "properties": { + "name": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "salary": { + "type": "number", + "description": "Annual salary", + "rls": { + "read": { "user_condition": { "role": "hr" } }, + "write": { "user_condition": { "role": "hr" } } + } + }, + "performance_notes": { + "type": "string", + "description": "Manager notes", + "rls": { + "read": { + "$or": [ + { "user_condition": { "role": "manager" } }, + { "user_condition": { "role": "hr" } } + ] + }, + "write": { "user_condition": { "role": "manager" } } + } + } + } +} +``` + +### Admin-Only Internal Fields + +```jsonc +{ + "name": "Order", + "type": "object", + "properties": { + "order_number": { "type": "string" }, + "total": { "type": "number" }, + "internal_notes": { + "type": "string", + "description": "Internal processing notes", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": { "user_condition": { "role": "admin" } } + } + }, + "profit_margin": { + "type": "number", + "description": "Profit margin percentage", + "rls": { + "read": { "user_condition": { "role": "admin" } }, + "write": false + } + } + } +} +``` + +--- + +## Complex Patterns (Dashboard UI or Backend) + +Some patterns may still require the Dashboard UI or backend functions. + +### Bidirectional Relationships (e.g., Friendships, Matches) + +**Requirement:** Either party in a relationship should have access. + +**Now possible with $or:** +```jsonc +{ + "rls": { + "read": { + "$or": [ + { "data.user_a_email": "{{user.email}}" }, + { "data.user_b_email": "{{user.email}}" } + ] + } + } +} +``` + +**Alternative solutions:** +1. **Entity redesign:** Store two records per relationship (one for each party) +2. **Backend function:** Query with custom logic + +### Complex Business Logic + +**Requirement:** Access depends on multiple entity fields with complex conditions. + +**JSON Schema limitation:** While operators help, very complex business logic may still be hard to express. + +**Solution options:** +1. **Backend function:** Implement custom access logic +2. **Combine simpler rules:** Break complex rules into simpler entity-level and field-level rules + +--- + +## Best Practices + +### Security Strategy + +Use a combination of entity-level RLS and field-level security: + +| Data Type | Approach | Example | +|-----------|----------|---------| +| User-editable | Entity RLS: Owner-only | UserProfile with `created_by` check | +| Sensitive fields | Field-level RLS | Salary field with HR role check | +| Multi-role access | `$or` with user_condition | Admin OR Manager access | +| Conditional access | Field operators | `$in`, `$ne` on data fields | +| Public content | Entity RLS: `read: true` | PublicPost | +| Private content | Entity RLS: Owner-only | PrivateNote | + +### When to Use Each Approach + +| Requirement | Approach | +|-------------|----------| +| Single condition (owner, admin, department) | JSON Schema RLS | +| Multiple OR/AND conditions | JSON Schema RLS with `$or`/`$and` | +| Field value checks with `$in`/`$ne`/etc. | JSON Schema RLS for `data.*` fields | +| Field-level access control | JSON Schema FLS (field-level `rls`) | +| Complex comparison operators (`$gt`, `$lt`) | Backend functions | +| Very complex business logic | Backend functions | + +### Common Role Patterns + +| Role | Typical Access | +|------|----------------| +| `admin` | Full access to all records | +| `moderator` | Read/update access, limited delete | +| `manager` | Department-scoped access | +| `user` | Own records only | + +### Supported Operators Summary + +| Operator | Supported | Notes | +|----------|-----------|-------| +| `$or` | Yes | Combine multiple conditions | +| `$and` | Yes | All conditions must match | +| `$nor` | Yes | None of the conditions match | +| `$in` | Yes | For `data.*` fields only | +| `$nin` | Yes | For `data.*` fields only | +| `$ne` | Yes | For `data.*` fields only | +| `$all` | Yes | For `data.*` fields only | +| `$gt`, `$lt`, `$gte`, `$lte` | No | Use backend functions | +| `$regex` | No | Use backend functions | + +### Limitations Summary + +| Not Supported | Alternative | +|---------------|-------------| +| Operators on `user_condition` | Use equality only for user checks | +| Comparison operators (`$gt`, `$lt`) | Backend functions | +| Regex matching (`$regex`) | Backend functions | +| Cross-entity relationships | Backend functions | diff --git a/evals/fixtures/skill-check/skills/base44-cli/references/site-deploy.md b/evals/fixtures/skill-check/skills/base44-cli/references/site-deploy.md new file mode 100644 index 0000000..929d302 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-cli/references/site-deploy.md @@ -0,0 +1,118 @@ +# base44 site deploy + +Deploy built site files to Base44 hosting. + +## Table of Contents + +- [Syntax](#syntax) +- [Authentication](#authentication) +- [Prerequisites](#prerequisites) +- [How It Works](#how-it-works) +- [Interactive Flow](#interactive-flow) +- [Typical Workflow](#typical-workflow) +- [Configuration](#configuration) +- [Error Handling](#error-handling) +- [Use Cases](#use-cases) +- [Notes](#notes) + +## Syntax + +```bash +npx base44 site deploy [options] +``` + +## Options + +| Option | Description | +| ------------ | ------------------------- | +| `-y, --yes` | Skip confirmation prompt | + +Use `-y` flag for non-interactive/automated deployments: + +```bash +npx base44 site deploy -y +``` + +## Authentication + +**Required**: Yes. If not authenticated, you'll be prompted to login first. + +## Prerequisites + +- Must be run from a Base44 project directory +- Project must have `site.outputDirectory` configured in project config +- Site must be built before deploying (run your build command first) +- **SPA only**: Base44 hosting supports Single Page Applications with a single `index.html` entry point. All routes are served from `index.html` (client-side routing). + +## How It Works + +1. Reads project configuration +2. Validates that site configuration exists +3. Prompts for deployment confirmation showing the output directory +4. Creates an archive of site files from the output directory +5. Deploys to Base44 hosting +6. Returns the app URL + +## Interactive Flow + +```bash +$ npx base44 site deploy + +Deploy site from ./dist? (yes/no) yes + +Creating archive... +Uploading to Base44... +Deploying... + +✓ Deployment successful! + +Visit your site at: https://my-app.base44.app +``` + +## Typical Workflow + +```bash +# 1. Build your site using your framework's build command +npm run build + +# 2. Deploy to Base44 +npx base44 site deploy +``` + +## Configuration + +The `site.outputDirectory` in your project configuration should point to where your framework outputs built files: + +- Vite: typically `./dist` +- Next.js: typically `./.next` or `./out` +- Create React App: typically `./build` +- Custom: whatever your build tool outputs to + +## Error Handling + +If site configuration is missing: +```bash +$ npx base44 site deploy +Error: No site configuration found in project +``` + +If you cancel the deployment: +```bash +Deploy site from ./dist? (yes/no) no +Deployment cancelled +``` + +## Use Cases + +- Deploy your site after making changes +- Push new versions of your application +- Deploy after updating content or functionality +- Part of your CI/CD pipeline + +## Notes + +- Always build your site before deploying +- The command deploys whatever is in your output directory +- Make sure your build completed successfully before deploying +- Previous deployments are preserved (versioned) in Base44 +- Deployment is immediate and updates your live site diff --git a/evals/fixtures/skill-check/skills/base44-sdk/SKILL.md b/evals/fixtures/skill-check/skills/base44-sdk/SKILL.md new file mode 100644 index 0000000..01ddd7f --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/SKILL.md @@ -0,0 +1,296 @@ +--- +name: base44-sdk +description: "The base44 SDK is the library to communicate with base44 services. In projects, you use it to communicate with remote resources (entities, backend functions, ai agents) and to write backend functions. This skill is the place for learning about available modules and types. When you plan or implement a feature, you must learn this skill" +--- + +# Base44 Coder + +Build apps on the Base44 platform using the Base44 JavaScript SDK. + +## ⚡ IMMEDIATE ACTION REQUIRED - Read This First + +This skill activates on ANY mention of "base44" or when a `base44/` folder exists. **DO NOT read documentation files or search the web before acting.** + +**Your first action MUST be:** +1. Check if `base44/config.jsonc` exists in the current directory +2. If **YES** (existing project scenario): + - This skill (base44-sdk) handles the request + - Implement features using Base44 SDK + - Do NOT use base44-cli unless user explicitly requests CLI commands +3. If **NO** (new project scenario): + - Transfer to base44-cli skill for project initialization + - This skill cannot help until project is initialized + +## When to Use This Skill vs base44-cli + +**Use base44-sdk when:** +- Building features in an **EXISTING** Base44 project +- `base44/config.jsonc` already exists in the project +- Base44 SDK imports are present (`@base44/sdk`) +- Writing JavaScript/TypeScript code using Base44 SDK modules +- Implementing functionality, components, or features +- User mentions: "implement", "build a feature", "add functionality", "write code for" +- User says "create a [type] app" **and** a Base44 project already exists + +**DO NOT USE base44-sdk for:** +- ❌ Initializing new Base44 projects (use `base44-cli` instead) +- ❌ Empty directories without Base44 configuration +- ❌ When user says "create a new Base44 project/app/site" and no project exists +- ❌ CLI commands like `npx base44 create`, `npx base44 deploy`, `npx base44 login` (use `base44-cli`) + +**Skill Dependencies:** +- `base44-sdk` assumes a Base44 project is **already initialized** +- `base44-cli` is a **prerequisite** for `base44-sdk` in new projects +- If user wants to "create an app" and no Base44 project exists, use `base44-cli` first + +**State Check Logic:** +Before selecting this skill, verify: +- IF (user mentions "create/build app" OR "make a project"): + - IF (directory is empty OR no `base44/config.jsonc` exists): + → Use **base44-cli** (project initialization needed) + - ELSE: + → Use **base44-sdk** (project exists, build features) + +## Quick Start + +```javascript +// In Base44-generated apps, base44 client is pre-configured and available + +// CRUD operations +const task = await base44.entities.Task.create({ title: "New task", status: "pending" }); +const tasks = await base44.entities.Task.list(); +await base44.entities.Task.update(task.id, { status: "done" }); + +// Get current user +const user = await base44.auth.me(); +``` + +```javascript +// External apps +import { createClient } from "@base44/sdk"; + +// IMPORTANT: Use 'appId' (NOT 'clientId' or 'id') +const base44 = createClient({ appId: "your-app-id" }); +await base44.auth.loginViaEmailPassword("user@example.com", "password"); +``` + +## ⚠️ CRITICAL: Do Not Hallucinate APIs + +**Before writing ANY Base44 code, verify method names against this table or [QUICK_REFERENCE.md](references/QUICK_REFERENCE.md).** + +Base44 SDK has unique method names. Do NOT assume patterns from Firebase, Supabase, or other SDKs. + +### Authentication - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `signInWithGoogle()` | `loginWithProvider('google')` | +| `signInWithProvider('google')` | `loginWithProvider('google')` | +| `auth.google()` | `loginWithProvider('google')` | +| `signInWithEmailAndPassword(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `signIn(email, pw)` | `loginViaEmailPassword(email, pw)` | +| `createUser()` / `signUp()` | `register({email, password})` | +| `onAuthStateChanged()` | `me()` (no listener, call when needed) | +| `currentUser` | `await auth.me()` | + +### Functions - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `functions.call('name', data)` | `functions.invoke('name', data)` | +| `functions.run('name', data)` | `functions.invoke('name', data)` | +| `callFunction('name', data)` | `functions.invoke('name', data)` | +| `httpsCallable('name')(data)` | `functions.invoke('name', data)` | + +### Integrations - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `ai.generate(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `openai.chat(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `llm(prompt)` | `integrations.Core.InvokeLLM({prompt})` | +| `sendEmail(to, subject, body)` | `integrations.Core.SendEmail({to, subject, body})` | +| `email.send()` | `integrations.Core.SendEmail({to, subject, body})` | +| `uploadFile(file)` | `integrations.Core.UploadFile({file})` | +| `storage.upload(file)` | `integrations.Core.UploadFile({file})` | + +### Entities - WRONG vs CORRECT + +| ❌ WRONG (hallucinated) | ✅ CORRECT | +|------------------------|-----------| +| `entities.Task.find({...})` | `entities.Task.filter({...})` | +| `entities.Task.findOne(id)` | `entities.Task.get(id)` | +| `entities.Task.insert(data)` | `entities.Task.create(data)` | +| `entities.Task.remove(id)` | `entities.Task.delete(id)` | +| `entities.Task.onChange(cb)` | `entities.Task.subscribe(cb)` | + +## SDK Modules + +| Module | Purpose | Reference | +|--------|---------|-----------| +| `entities` | CRUD operations on data models | [entities.md](references/entities.md) | +| `auth` | Login, register, user management | [auth.md](references/auth.md) | +| `agents` | AI conversations and messages | [base44-agents.md](references/base44-agents.md) | +| `functions` | Backend function invocation | [functions.md](references/functions.md) | +| `integrations` | AI, email, file uploads, custom APIs | [integrations.md](references/integrations.md) | +| `connectors` | OAuth tokens (service role only) | [connectors.md](references/connectors.md) | +| `analytics` | Track custom events and user activity | [analytics.md](references/analytics.md) | +| `appLogs` | Log user activity in app | [app-logs.md](references/app-logs.md) | +| `users` | Invite users to the app | [users.md](references/users.md) | + +For client setup and authentication modes, see [client.md](references/client.md). + +**TypeScript Support:** Each reference file includes a "Type Definitions" section with TypeScript interfaces and types for the module's methods, parameters, and return values. + +## Installation + +Install the Base44 SDK: + +```bash +npm install @base44/sdk +``` + +**Important:** Never assume or hardcode the `@base44/sdk` package version. Always install without a version specifier to get the latest version. + +## Creating a Client (External Apps) + +When creating a client in external apps, **ALWAYS use `appId` as the parameter name**: + +```javascript +import { createClient } from "@base44/sdk"; + +// ✅ CORRECT +const base44 = createClient({ appId: "your-app-id" }); + +// ❌ WRONG - Do NOT use these: +// const base44 = createClient({ clientId: "your-app-id" }); // WRONG +// const base44 = createClient({ id: "your-app-id" }); // WRONG +``` + +**Required parameter:** `appId` (string) - Your Base44 application ID + +**Optional parameters:** +- `token` (string) - Pre-authenticated user token +- `options` (object) - Configuration options + - `options.onError` (function) - Global error handler + +**Example with error handler:** +```javascript +const base44 = createClient({ + appId: "your-app-id", + options: { + onError: (error) => { + console.error("Base44 error:", error); + } + } +}); +``` + +## Module Selection + +**Working with app data?** +- Create/read/update/delete records → `entities` +- Import data from file → `entities.importEntities()` +- Realtime updates → `entities.EntityName.subscribe()` + +**User management?** +- Login/register/logout → `auth` +- Get current user → `auth.me()` +- Update user profile → `auth.updateMe()` +- Invite users → `users.inviteUser()` + +**AI features?** +- Chat with AI agents → `agents` (requires logged-in user) +- Create new conversation → `agents.createConversation()` +- Manage conversations → `agents.getConversations()` +- Generate text/JSON with AI → `integrations.Core.InvokeLLM()` +- Generate images → `integrations.Core.GenerateImage()` + +**Custom backend logic?** +- Run server-side code → `functions.invoke()` +- Need admin access → `base44.asServiceRole.functions.invoke()` + +**External services?** +- Send emails → `integrations.Core.SendEmail()` +- Upload files → `integrations.Core.UploadFile()` +- Custom APIs → `integrations.custom.call()` +- OAuth tokens (Google, Slack) → `connectors` (backend only) + +**Tracking and analytics?** +- Track custom events → `analytics.track()` +- Log page views/activity → `appLogs.logUserInApp()` + +## Common Patterns + +### Filter and Sort Data + +```javascript +const pendingTasks = await base44.entities.Task.filter( + { status: "pending", assignedTo: userId }, // query + "-created_date", // sort (descending) + 10, // limit + 0 // skip +); +``` + +### Protected Routes (check auth) + +```javascript +const user = await base44.auth.me(); +if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return; +} +``` + +### Backend Function Call + +```javascript +// Frontend +const result = await base44.functions.invoke("processOrder", { + orderId: "123", + action: "ship" +}); + +// Backend function (Deno) +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const { orderId, action } = await req.json(); + // Process with service role for admin access + const order = await base44.asServiceRole.entities.Orders.get(orderId); + return Response.json({ success: true }); +}); +``` + +### Service Role Access + +Use `asServiceRole` in backend functions for admin-level operations: + +```javascript +// User mode - respects permissions +const myTasks = await base44.entities.Task.list(); + +// Service role - full access (backend only) +const allTasks = await base44.asServiceRole.entities.Task.list(); +const token = await base44.asServiceRole.connectors.getAccessToken("slack"); +``` + +## Frontend vs Backend + +| Capability | Frontend | Backend | +|------------|----------|---------| +| `entities` (user's data) | Yes | Yes | +| `auth` | Yes | Yes | +| `agents` | Yes | Yes | +| `functions.invoke()` | Yes | Yes | +| `integrations` | Yes | Yes | +| `analytics` | Yes | Yes | +| `appLogs` | Yes | Yes | +| `users` | Yes | Yes | +| `asServiceRole.*` | No | Yes | +| `connectors` | No | Yes | + +Backend functions use `Deno.serve()` and `createClientFromRequest(req)` to get a properly authenticated client. diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/QUICK_REFERENCE.md b/evals/fixtures/skill-check/skills/base44-sdk/references/QUICK_REFERENCE.md new file mode 100644 index 0000000..d357384 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/QUICK_REFERENCE.md @@ -0,0 +1,155 @@ +# Base44 SDK Quick Reference + +Compact method signatures for all SDK modules. **Verify against this before writing code.** + +--- + +## Auth (`base44.auth.*`) + +``` +loginViaEmailPassword(email, password, turnstileToken?) → Promise<{access_token, user}> +loginWithProvider('google' | 'microsoft' | 'facebook', fromUrl?) → void +me() → Promise +updateMe(data) → Promise +isAuthenticated() → Promise +logout(redirectUrl?) → void +redirectToLogin(nextUrl) → void # ⚠️ Avoid - prefer custom login UI +register({email, password, turnstile_token?, referral_code?}) → Promise +verifyOtp({email, otpCode}) → Promise +resendOtp(email) → Promise +inviteUser(userEmail, role) → Promise +resetPasswordRequest(email) → Promise +resetPassword({resetToken, newPassword}) → Promise +changePassword({userId, currentPassword, newPassword}) → Promise +setToken(token, saveToStorage?) → void +``` + +--- + +## Entities (`base44.entities.EntityName.*`) + +``` +create(data) → Promise +bulkCreate(dataArray) → Promise +list(sort?, limit?, skip?, fields?) → Promise +filter(query, sort?, limit?, skip?, fields?) → Promise +get(id) → Promise +update(id, data) → Promise +delete(id) → Promise +deleteMany(query) → Promise +importEntities(file) → Promise // frontend only +subscribe(callback) → () => void // returns unsubscribe fn +``` + +**Sort:** Use `-fieldName` for descending (e.g., `-created_date`) + +--- + +## Functions (`base44.functions.*`) + +``` +invoke(functionName, data) → Promise +``` + +**Backend:** Use `base44.asServiceRole.functions.invoke()` for admin access. + +--- + +## Integrations (`base44.integrations.Core.*`) + +``` +InvokeLLM({prompt, add_context_from_internet?, response_json_schema?, file_urls?}) → Promise +GenerateImage({prompt}) → Promise<{url}> +SendEmail({to, subject, body, from_name?}) → Promise +UploadFile({file}) → Promise<{file_url}> +UploadPrivateFile({file}) → Promise<{file_uri}> +CreateFileSignedUrl({file_uri, expires_in?}) → Promise<{signed_url}> +ExtractDataFromUploadedFile({file_url, json_schema}) → Promise +``` + +### Custom Integrations (`base44.integrations.custom.*`) + +``` +call(slug, operationId, {payload?, pathParams?, queryParams?, headers?}?) → Promise<{success, status_code, data}> +``` + +**operationId format:** `"method:/path"` (e.g., `"get:/contacts"`, `"post:/users/{id}"`) + +--- + +## Analytics (`base44.analytics.*`) + +``` +track({eventName, properties?}) → void +``` + +--- + +## App Logs (`base44.appLogs.*`) + +``` +logUserInApp(pageName) → Promise +``` + +--- + +## Users (`base44.users.*`) + +``` +inviteUser(userEmail, role) → Promise // role: 'user' | 'admin' +``` + +--- + +## Connectors (`base44.asServiceRole.connectors.*`) + +**Backend only, service role required.** + +``` +getAccessToken(integrationType) → Promise +``` + +**Types:** `'googlecalendar'`, `'googledrive'`, `'slack'`, `'notion'`, `'salesforce'`, `'hubspot'`, `'linkedin'`, `'tiktok'`, `'github'` + +--- + +## Service Role Access + +**Backend functions only.** Prefix any module with `asServiceRole` for admin access: + +```javascript +base44.asServiceRole.entities.Task.list() +base44.asServiceRole.functions.invoke('name', data) +base44.asServiceRole.connectors.getAccessToken('slack') +``` + +--- + +## Backend Function Template + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + const data = await req.json(); + + // User context + const user = await base44.auth.me(); + + // Service role for admin operations + const allRecords = await base44.asServiceRole.entities.Task.list(); + + return Response.json({ success: true }); +}); +``` + +--- + +## Client Initialization (External Apps) + +```javascript +import { createClient } from "@base44/sdk"; + +const base44 = createClient({ appId: "your-app-id" }); // MUST use 'appId' +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/analytics.md b/evals/fixtures/skill-check/skills/base44-sdk/references/analytics.md new file mode 100644 index 0000000..d1b2c69 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/analytics.md @@ -0,0 +1,113 @@ +# Analytics Module + +Track custom events and user activity via `base44.analytics`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Automatic Tracking](#automatic-tracking) +- [Best Practices](#best-practices) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `track(params)` | `void` | Track a custom event | + +## Examples + +### Track Custom Event + +```javascript +// Track a simple event +base44.analytics.track({ + eventName: "button_clicked" +}); + +// Track event with properties +base44.analytics.track({ + eventName: "purchase_completed", + properties: { + product_id: "prod-123", + amount: 99.99, + currency: "USD" + } +}); +``` + +### Track User Actions + +```javascript +// Track page view +base44.analytics.track({ + eventName: "page_view", + properties: { + page: "/dashboard", + referrer: document.referrer + } +}); + +// Track feature usage +base44.analytics.track({ + eventName: "feature_used", + properties: { + feature: "export_data", + format: "csv" + } +}); +``` + +## Automatic Tracking + +The analytics module automatically tracks: +- **Initialization events**: When the app loads +- **Heartbeat events**: Periodic activity signals +- **Session duration**: Time spent in the app + +These internal events help measure user engagement without manual instrumentation. + +## Best Practices + +1. **Use descriptive event names**: `order_completed` instead of `click` +2. **Include relevant properties**: Add context that helps analyze the event +3. **Be consistent**: Use the same event names and property keys across your app +4. **Don't track sensitive data**: Avoid PII in event properties + +```javascript +// Good: Descriptive with relevant properties +base44.analytics.track({ + eventName: "subscription_started", + properties: { + plan: "pro", + billing_cycle: "annual" + } +}); + +// Avoid: Vague event name, no context +base44.analytics.track({ + eventName: "click" +}); +``` + +## Type Definitions + +```typescript +/** Properties that can be attached to a tracked event. */ +type TrackEventProperties = { + [key: string]: string | number | boolean | null | undefined; +}; + +/** Parameters for the track() method. */ +interface TrackEventParams { + /** The name of the event to track. */ + eventName: string; + /** Optional properties to attach to the event. */ + properties?: TrackEventProperties; +} + +/** The analytics module interface. */ +interface AnalyticsModule { + /** Track a custom event with optional properties. */ + track(params: TrackEventParams): void; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/app-logs.md b/evals/fixtures/skill-check/skills/base44-sdk/references/app-logs.md new file mode 100644 index 0000000..0439805 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/app-logs.md @@ -0,0 +1,78 @@ +# App Logs Module + +Log user activity in your app via `base44.appLogs`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) +- [Use Cases](#use-cases) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `logUserInApp(pageName)` | `Promise` | Log user activity on a page | + +## Examples + +### Log User Activity + +```javascript +// Log when user visits a page +await base44.appLogs.logUserInApp("dashboard"); + +// Log specific page visits +await base44.appLogs.logUserInApp("settings"); +await base44.appLogs.logUserInApp("profile"); + +// Log feature usage +await base44.appLogs.logUserInApp("export-button-click"); +``` + +The page name doesn't have to be an actual page - it can be any string you want to track. + +## Use Cases + +### Track Page Views in React + +```javascript +// Log page views on route change +useEffect(() => { + base44.appLogs.logUserInApp(window.location.pathname); +}, [location.pathname]); +``` + +### Track Feature Usage + +```javascript +// Log when user uses specific features +function handleExport() { + base44.appLogs.logUserInApp("export-data"); + // ... export logic +} + +function handleSettingsChange() { + base44.appLogs.logUserInApp("settings-updated"); + // ... save settings +} +``` + +## Notes + +- Logs appear in the Analytics page of your app dashboard +- App logs track page-level and feature-level activity +- Use `analytics.track()` for custom events with properties, `appLogs.logUserInApp()` for simple page/feature tracking + +## Type Definitions + +```typescript +/** App Logs module for tracking and analyzing app usage. */ +interface AppLogsModule { + /** + * Log user activity in the app. + * @param pageName - Name of the page or section being visited. + * @returns Promise that resolves when the log is recorded. + */ + logUserInApp(pageName: string): Promise; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/auth.md b/evals/fixtures/skill-check/skills/base44-sdk/references/auth.md new file mode 100644 index 0000000..5195b3a --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/auth.md @@ -0,0 +1,761 @@ +# Auth Module + +User authentication, registration, and session management via `base44.auth`. + +## Contents +- [TypeScript Types](#typescript-types) +- [Methods](#methods) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Auth Providers](#auth-providers) +- [Environment Availability](#environment-availability) +- [App Visibility](#app-visibility) +- [Limitations](#limitations) + +--- + +## TypeScript Types + +### User Interface +```typescript +interface User { + id: string; + created_date: string; + updated_date: string; + email: string; + full_name: string | null; + disabled: boolean | null; + is_verified: boolean; + app_id: string; + is_service: boolean; + role: string; + [key: string]: any; // Custom schema fields +} +``` + +### LoginResponse Interface +```typescript +interface LoginResponse { + access_token: string; // JWT token + user: User; // Complete user object +} +``` + +### Parameter Interfaces + +#### RegisterParams +```typescript +interface RegisterParams { + email: string; // Required + password: string; // Required + turnstile_token?: string | null; // Optional: Cloudflare Turnstile for bot protection + referral_code?: string | null; // Optional: Referral code +} +``` + +#### VerifyOtpParams +```typescript +interface VerifyOtpParams { + email: string; // User's email + otpCode: string; // OTP code from email +} +``` + +#### ResetPasswordParams +```typescript +interface ResetPasswordParams { + resetToken: string; // Token from password reset email + newPassword: string; // New password to set +} +``` + +#### ChangePasswordParams +```typescript +interface ChangePasswordParams { + userId: string; // User ID + currentPassword: string; // Current password for verification + newPassword: string; // New password to set +} +``` + +### Provider Type +```typescript +type Provider = 'google' | 'microsoft' | 'facebook'; +``` + +--- + +## Methods + +### Module Interface +```typescript +interface AuthModule { + // User Info + me(): Promise; + updateMe(data: Partial>): Promise; + isAuthenticated(): Promise; + + // Login/Logout + loginViaEmailPassword(email: string, password: string, turnstileToken?: string): Promise; + loginWithProvider(provider: Provider, fromUrl?: string): void; + logout(redirectUrl?: string): void; + redirectToLogin(nextUrl: string): void; + + // Token Management + setToken(token: string, saveToStorage?: boolean): void; + + // Registration + register(params: RegisterParams): Promise; + verifyOtp(params: VerifyOtpParams): Promise; + resendOtp(email: string): Promise; + + // User Management + inviteUser(userEmail: string, role: string): Promise; + + // Password Management + resetPasswordRequest(email: string): Promise; + resetPassword(params: ResetPasswordParams): Promise; + changePassword(params: ChangePasswordParams): Promise; +} +``` + +### Method Reference Table + +| Method | Parameters | Return Type | Description | +|--------|-----------|-------------|-------------| +| `register()` | `params: RegisterParams` | `Promise` | Create new user account | +| `loginViaEmailPassword()` | `email: string, password: string, turnstileToken?: string` | `Promise` | Authenticate with email/password | +| `loginWithProvider()` | `provider: Provider, fromUrl?: string` | `void` | Initiate OAuth login flow | +| `me()` | None | `Promise` | Get current authenticated user | +| `updateMe()` | `data: Partial` | `Promise` | Update current user's profile | +| `logout()` | `redirectUrl?: string` | `void` | Clear session, optionally redirect | +| `redirectToLogin()` | `nextUrl: string` | `void` | ⚠️ **Avoid** - Prefer custom login UI with `loginViaEmailPassword()` or `loginWithProvider()` | +| `isAuthenticated()` | None | `Promise` | Check if user is logged in | +| `setToken()` | `token: string, saveToStorage?: boolean` | `void` | Manually set auth token | +| `inviteUser()` | `userEmail: string, role: string` | `Promise` | Send invitation email | +| `verifyOtp()` | `params: VerifyOtpParams` | `Promise` | Verify OTP code | +| `resendOtp()` | `email: string` | `Promise` | Resend OTP code | +| `resetPasswordRequest()` | `email: string` | `Promise` | Request password reset | +| `resetPassword()` | `params: ResetPasswordParams` | `Promise` | Reset password with token | +| `changePassword()` | `params: ChangePasswordParams` | `Promise` | Change user password | + +--- + +## Examples + +### Register New User (Complete Flow) + +Registration requires email verification before login. Complete flow: + +1. **Register** - Create the user account +2. **Verification email sent** - User receives an OTP code +3. **Verify OTP** - User enters code to verify email +4. **Login** - User can now log in + +```javascript +try { + // Step 1: Register the user + await base44.auth.register({ + email: "user@example.com", + password: "securePassword123", + referral_code: "OPTIONAL_CODE", // optional + turnstile_token: "CAPTCHA_TOKEN" // optional, for bot protection + }); + console.log('Registration successful. Check email for OTP code.'); + + // Step 2: User receives email with OTP code (e.g., "123456") + + // Step 3: Verify the OTP code + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" // code from verification email + }); + console.log('Email verified successfully.'); + + // Step 4: Now the user can log in + const loginResponse = await base44.auth.loginViaEmailPassword( + "user@example.com", + "securePassword123" + ); + console.log('Login successful:', loginResponse.user); + +} catch (error) { + console.error('Registration flow failed:', error.message); + // Handle specific errors (see Error Handling section) +} +``` + +> **Important**: Users cannot log in until they complete OTP verification. Attempting to call `loginViaEmailPassword` before verification will fail. + +### Login with Email/Password + +```javascript +try { + const response = await base44.auth.loginViaEmailPassword( + "user@example.com", + "password123", + turnstileToken // optional: for bot protection + ); + + console.log('Login successful'); + console.log('User:', response.user); + console.log('Token:', response.access_token); + + // JWT is automatically stored for subsequent requests + +} catch (error) { + console.error('Login failed:', error.message); + if (error.status === 401) { + console.error('Invalid credentials'); + } else if (error.status === 403) { + console.error('Email not verified. Please check your email for OTP.'); + } +} +``` + +### Login with OAuth Provider + +```javascript +// Redirect to Google OAuth +base44.auth.loginWithProvider('google'); + +// Redirect to Google OAuth and return to current page after +base44.auth.loginWithProvider('google', window.location.href); + +// Supported providers: 'google', 'microsoft', 'facebook' +base44.auth.loginWithProvider('microsoft'); +``` + +### Get Current User + +```javascript +try { + const user = await base44.auth.me(); + + if (user) { + console.log('User ID:', user.id); + console.log('Email:', user.email); + console.log('Name:', user.full_name); + console.log('Role:', user.role); + console.log('Verified:', user.is_verified); + } else { + console.log('User not authenticated'); + } + +} catch (error) { + console.error('Failed to fetch user:', error.message); + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } +} +``` + +### Update User Profile + +```javascript +try { + const updatedUser = await base44.auth.updateMe({ + full_name: "John Doe", + // Custom schema fields can be updated here + phone: "+1234567890", + preferences: { theme: "dark" } + }); + + console.log('Profile updated:', updatedUser); + +} catch (error) { + console.error('Profile update failed:', error.message); + if (error.status === 400) { + console.error('Invalid data provided'); + } else if (error.status === 401) { + console.error('Authentication required'); + navigate('/login'); + } +} +``` + +### Check Authentication Status + +```javascript +try { + const isLoggedIn = await base44.auth.isAuthenticated(); + + if (isLoggedIn) { + // Show authenticated UI + console.log('User is authenticated'); + } else { + // Show login button + console.log('User is not authenticated'); + } + +} catch (error) { + console.error('Failed to check authentication:', error.message); + // On error, treat as not authenticated +} +``` + +### Logout + +```javascript +// Simple logout +base44.auth.logout(); + +// Logout and redirect to goodbye page +base44.auth.logout("/goodbye"); + +// Logout and redirect to homepage +base44.auth.logout("/"); +``` + +### Protected Route Pattern + +```javascript +// Using a navigation function (e.g., React Router's useNavigate, Next.js router) +async function requireAuth(navigate) { + try { + const user = await base44.auth.me(); + + if (!user) { + // Navigate to your custom login page + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } + + return user; + + } catch (error) { + console.error('Authentication check failed:', error.message); + navigate('/login', { state: { returnTo: window.location.pathname } }); + return null; + } +} + +// Usage in your app +async function loadProtectedPage(navigate) { + const user = await requireAuth(navigate); + if (!user) { + // Will navigate to login + return; + } + + // Continue with authenticated logic + console.log('Welcome,', user.full_name); +} +``` + +### Set Authentication Token + +```javascript +// SECURITY WARNING: Never hardcode tokens or expose them in client code +// Tokens should only be received from secure authentication flows + +// Set token and save to localStorage (default) +base44.auth.setToken(receivedToken); + +// Set token without saving to localStorage (temporary session) +base44.auth.setToken(receivedToken, false); + +// Verify token was set +try { + const isAuthenticated = await base44.auth.isAuthenticated(); + if (!isAuthenticated) { + console.error('Token validation failed'); + } +} catch (error) { + console.error('Failed to set token:', error.message); +} +``` + +### Invite User to Application + +```javascript +try { + // Note: Typically requires admin privileges + const response = await base44.auth.inviteUser( + "newuser@example.com", + "user" // or "admin" + ); + + console.log('Invitation sent successfully'); + +} catch (error) { + console.error('Failed to invite user:', error.message); + if (error.status === 403) { + console.error('Insufficient permissions to invite users'); + } else if (error.status === 400) { + console.error('Invalid email or role'); + } +} +``` + +### OTP Verification + +```javascript +try { + // Verify OTP code sent to user's email + await base44.auth.verifyOtp({ + email: "user@example.com", + otpCode: "123456" + }); + + console.log('OTP verified successfully'); + +} catch (error) { + console.error('OTP verification failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired OTP code'); + } else if (error.status === 429) { + console.error('Too many attempts. Please try again later.'); + } +} + +// Resend OTP if needed +try { + await base44.auth.resendOtp("user@example.com"); + console.log('OTP resent successfully'); + +} catch (error) { + console.error('Failed to resend OTP:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + } +} +``` + +### Password Reset Flow + +```javascript +// Step 1: Request password reset +try { + await base44.auth.resetPasswordRequest("user@example.com"); + console.log('Password reset email sent. Check your inbox.'); + +} catch (error) { + console.error('Password reset request failed:', error.message); + if (error.status === 429) { + console.error('Too many requests. Please try again later.'); + } + // Note: For security, don't reveal if email exists +} + +// Step 2: Reset password with token from email +try { + await base44.auth.resetPassword({ + resetToken: "token-from-email", + newPassword: "newSecurePassword123" + }); + + console.log('Password reset successfully. You can now log in.'); + +} catch (error) { + console.error('Password reset failed:', error.message); + if (error.status === 400) { + console.error('Invalid or expired reset token'); + } else if (error.status === 422) { + console.error('Password does not meet requirements'); + } +} +``` + +### Change Password + +```javascript +try { + const currentUser = await base44.auth.me(); + + if (!currentUser) { + throw new Error('User must be authenticated to change password'); + } + + await base44.auth.changePassword({ + userId: currentUser.id, + currentPassword: "oldPassword123", + newPassword: "newSecurePassword456" + }); + + console.log('Password changed successfully'); + +} catch (error) { + console.error('Password change failed:', error.message); + if (error.status === 401) { + console.error('Current password is incorrect'); + } else if (error.status === 422) { + console.error('New password does not meet requirements'); + } else if (error.status === 403) { + console.error('Not authorized to change this password'); + } +} +``` + +--- + +## Error Handling + +### Common Error Scenarios + +The auth module can throw various errors. Here are common scenarios and how to handle them: + +#### Authentication Errors (401/403) +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Token expired or invalid - navigate to your custom login page + navigate('/login'); + } else if (error.status === 403) { + // Email not verified or insufficient permissions + console.error('Access denied:', error.message); + } +} +``` + +#### Validation Errors (400/422) +```javascript +try { + await base44.auth.register({ + email: "invalid-email", + password: "weak" + }); +} catch (error) { + if (error.status === 400) { + console.error('Invalid input:', error.message); + // Handle validation errors + } else if (error.status === 422) { + console.error('Data validation failed:', error.details); + } +} +``` + +#### Rate Limiting (429) +```javascript +try { + await base44.auth.resendOtp("user@example.com"); +} catch (error) { + if (error.status === 429) { + console.error('Too many requests. Please wait before trying again.'); + // Show countdown or disable button temporarily + } +} +``` + +#### Generic Error Handler +```javascript +function handleAuthError(error) { + switch (error.status) { + case 400: + return 'Invalid input. Please check your data.'; + case 401: + return 'Authentication required. Redirecting to login...'; + case 403: + return 'Access denied. You may need to verify your email.'; + case 404: + return 'Resource not found.'; + case 422: + return 'Validation failed. Please check your input.'; + case 429: + return 'Too many requests. Please try again later.'; + case 500: + return 'Server error. Please try again.'; + default: + return 'An unexpected error occurred.'; + } +} + +// Usage +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + console.error(handleAuthError(error)); +} +``` + +--- + +## Auth Providers + +Configure authentication providers in your app dashboard: + +### Available Providers + +**Built-in (All Plans):** +- **Email/Password** - Default, always enabled +- **Google** - OAuth authentication +- **Microsoft** - OAuth authentication +- **Facebook** - OAuth authentication + +**SSO Providers (Elite Plan):** +- **Okta** +- **Azure AD** +- **GitHub** + +### Using OAuth Providers + +```javascript +// Initiate OAuth login flow +base44.auth.loginWithProvider('google'); + +// Return to specific page after authentication +base44.auth.loginWithProvider('microsoft', '/dashboard'); + +// Supported values: 'google', 'microsoft', 'facebook' +``` + +--- + +## Environment Availability + +| Environment | Availability | Notes | +|------------|--------------|-------| +| **Frontend** | ✅ Yes | All methods available | +| **Backend Functions** | ✅ Yes | Use `createClientFromRequest(req)` for authenticated client | +| **Service Role** | ❌ No | Auth methods not available in service role context | + +### Frontend Usage +```javascript +// Standard browser usage +const user = await base44.auth.me(); +``` + +### Backend Functions Usage +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get user from request context + const user = await base44.auth.me(); + + // User operations here... + return Response.json({ user }); +}); +``` + +--- + +## App Visibility + +Control who can access your app in the app settings: + +### Public Apps +- No login required for basic access +- Users can view public content without authentication +- Authenticated users get additional features/data + +### Private Apps +- Login required to access any content +- Unauthenticated users are automatically redirected to login +- All content is protected by default + +--- + +## Limitations + +### Authentication UI Options +- **Recommended:** Build custom login/signup UI using `loginViaEmailPassword()` and `loginWithProvider()` for full control over user experience and branding +- **Alternative:** `redirectToLogin()` uses Base44's hosted authentication pages with limited customization + +### Hosted Login (via redirectToLogin) +- `redirectToLogin()` shows both login and signup options on the same page +- No separate `redirectToSignup()` method +- Users can switch between login/signup on the hosted page +- ⚠️ **Note:** Prefer building custom login UI for better user experience + +### Password Requirements +- Minimum length and complexity requirements enforced +- Requirements not exposed via API +- Validation errors returned when requirements not met + +### Rate Limiting +- OTP requests are rate-limited to prevent abuse +- Password reset requests are rate-limited +- Login attempts may be rate-limited with Turnstile protection + +### Token Management +- JWTs are automatically stored in localStorage by default +- Token expiration and refresh not exposed in API +- Call `me()` or `isAuthenticated()` to verify token validity + +--- + +## Best Practices + +### 1. Always Handle Errors +```javascript +try { + await base44.auth.loginViaEmailPassword(email, password); +} catch (error) { + // Always handle authentication errors + console.error('Login failed:', error.message); +} +``` + +### 2. Verify Authentication Before Protected Actions +```javascript +const user = await requireAuth(); +if (!user) return; // Will redirect to login +// Proceed with authenticated action +``` + +### 3. Use Type Safety with TypeScript +```typescript +import type { User, LoginResponse } from '@base44/sdk'; + +const user: User = await base44.auth.me(); +const response: LoginResponse = await base44.auth.loginViaEmailPassword(email, password); +``` + +### 4. Don't Hardcode Credentials +```javascript +// ❌ BAD +base44.auth.setToken("hardcoded-token-here"); + +// ✅ GOOD +const token = await secureAuthFlow(); +base44.auth.setToken(token); +``` + +### 5. Provide User Feedback +```javascript +try { + await base44.auth.register(params); + showSuccessMessage('Registration successful! Check your email for verification code.'); +} catch (error) { + showErrorMessage('Registration failed: ' + error.message); +} +``` + +### 6. Handle Token Expiration Gracefully +```javascript +try { + const user = await base44.auth.me(); +} catch (error) { + if (error.status === 401) { + // Clear local state and navigate to login + localStorage.clear(); + navigate('/login'); + } +} +``` + +### 7. Build Custom Login UI (Recommended) +```javascript +// ✅ RECOMMENDED - Custom login form with direct methods +const handleLogin = async (email, password) => { + try { + const { user } = await base44.auth.loginViaEmailPassword(email, password); + navigate('/dashboard'); + } catch (error) { + setError(error.message); + } +}; + +const handleGoogleLogin = () => { + base44.auth.loginWithProvider('google', '/dashboard'); +}; + +// ❌ AVOID - Redirecting to hosted login page +// base44.auth.redirectToLogin(window.location.href); +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/base44-agents.md b/evals/fixtures/skill-check/skills/base44-sdk/references/base44-agents.md new file mode 100644 index 0000000..ee55dee --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/base44-agents.md @@ -0,0 +1,364 @@ +# Agents Module + +AI agent conversations and messages via `base44.agents`. + +> **Note:** This module requires a logged-in user. All agent methods work in the context of the authenticated user. + +## Contents +- [Concepts](#concepts) +- [Methods](#methods) +- [Examples](#examples) (Create, Get Conversations, List, Subscribe, Send Message, WhatsApp) +- [Message Structure](#message-structure) +- [Conversation Structure](#conversation-structure) +- [Common Patterns](#common-patterns) + +## Concepts + +- **Conversation**: A dialogue between user and an AI agent. Has unique ID, agent name, user reference, and metadata. +- **Message**: Single message in a conversation. Has role (`user`, `assistant`, `system`), content, timestamps, and optional metadata. + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `createConversation(params)` | `Promise` | Create a new conversation with an agent | +| `getConversations()` | `Promise` | Get all user's conversations | +| `getConversation(id)` | `Promise` | Get conversation with messages | +| `listConversations(filterParams)` | `Promise` | Filter/sort/paginate conversations | +| `subscribeToConversation(id, onUpdate?)` | `() => void` | Real-time updates via WebSocket (returns unsubscribe function) | +| `addMessage(conversation, message)` | `Promise` | Send a message | +| `getWhatsAppConnectURL(agentName)` | `string` | Get WhatsApp connection URL for agent | + +## Examples + +### Create Conversation + +```javascript +const conversation = await base44.agents.createConversation({ + agent_name: "support-agent", + metadata: { + order_id: "ORD-123", + category: "billing" + } +}); + +console.log(conversation.id); +``` + +### Get All Conversations + +```javascript +const conversations = await base44.agents.getConversations(); + +conversations.forEach(conv => { + console.log(conv.id, conv.agent_name, conv.created_date); +}); +``` + +### Get Single Conversation (with messages) + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +console.log(conversation.messages); +``` + +### List with Filters + +```javascript +// Using filterParams object with q, sort, limit, skip, fields +const recent = await base44.agents.listConversations({ + q: { agent_name: "support-agent" }, + sort: "-created_date", + limit: 10, + skip: 0 +}); + +// Filter by metadata +const highPriority = await base44.agents.listConversations({ + q: { + agent_name: "support-agent", + "metadata.priority": "high" + }, + sort: "-updated_date", + limit: 20 +}); +``` + +### Subscribe to Updates (Real-time) + +```javascript +const unsubscribe = base44.agents.subscribeToConversation( + "conv-id-123", + (updatedConversation) => { + // Called when new messages arrive + console.log("New messages:", updatedConversation.messages); + } +); + +// Later: unsubscribe +unsubscribe(); +``` + +### Send a Message + +```javascript +const conversation = await base44.agents.getConversation("conv-id-123"); + +await base44.agents.addMessage(conversation, { + role: "user", + content: "What's the weather like today?" +}); +``` + +### Get WhatsApp Connection URL + +```javascript +const whatsappUrl = base44.agents.getWhatsAppConnectURL("support-agent"); +// Returns URL for users to connect with agent via WhatsApp +console.log(whatsappUrl); +``` + +## Message Structure + +```javascript +{ + role: "user" | "assistant" | "system", + content: "Message text or structured object", + created_date: "2024-01-15T10:30:00Z", + updated_date: "2024-01-15T10:30:00Z", + + // Optional fields + reasoning: { + content: "Agent's reasoning process", + timing: 1500 + }, + tool_calls: [{ + name: "search", + arguments: { query: "weather" }, + result: { ... }, + status: "success" + }], + file_urls: ["https://..."], + usage: { + prompt_tokens: 150, + completion_tokens: 50 + }, + metadata: { ... }, + custom_context: { ... } +} +``` + +## Conversation Structure + +```javascript +{ + id: "conv-id-123", + app_id: "app-id", + agent_name: "support-agent", + created_by_id: "user-id", + created_date: "2024-01-15T10:00:00Z", + updated_date: "2024-01-15T10:30:00Z", + messages: [ ... ], + metadata: { ... } +} +``` + +## Common Patterns + +### Chat Interface + +```javascript +// Load conversation +const conv = await base44.agents.getConversation(conversationId); +setMessages(conv.messages); + +// Subscribe to updates +const unsubscribe = base44.agents.subscribeToConversation(conversationId, (updated) => { + setMessages(updated.messages); +}); + +// Send message +async function sendMessage(text) { + await base44.agents.addMessage(conv, { role: "user", content: text }); +} + +// Cleanup on unmount +return () => unsubscribe(); +``` + +## Type Definitions + +### AgentConversation + +```typescript +/** An agent conversation containing messages exchanged with an AI agent. */ +interface AgentConversation { + /** Unique identifier for the conversation. */ + id: string; + /** Application ID. */ + app_id: string; + /** Name of the agent in this conversation. */ + agent_name: string; + /** ID of the user who created the conversation. */ + created_by_id: string; + /** When the conversation was created. */ + created_date: string; + /** When the conversation was last updated. */ + updated_date: string; + /** Array of messages in the conversation. */ + messages: AgentMessage[]; + /** Optional metadata associated with the conversation. */ + metadata?: Record; +} +``` + +### AgentMessage + +```typescript +/** A message in an agent conversation. */ +interface AgentMessage { + /** Unique identifier for the message. */ + id: string; + /** Role of the message sender. */ + role: "user" | "assistant" | "system"; + /** When the message was created. */ + created_date: string; + /** When the message was last updated. */ + updated_date: string; + /** Message content. */ + content?: string | Record; + /** Optional reasoning information for the message. */ + reasoning?: AgentMessageReasoning | null; + /** URLs to files attached to the message. */ + file_urls?: string[]; + /** Tool calls made by the agent. */ + tool_calls?: AgentMessageToolCall[]; + /** Token usage statistics. */ + usage?: AgentMessageUsage; + /** Whether the message is hidden from the user. */ + hidden?: boolean; + /** Custom context provided with the message. */ + custom_context?: AgentMessageCustomContext[]; + /** Model used to generate the message. */ + model?: string; + /** Checkpoint ID for the message. */ + checkpoint_id?: string; + /** Metadata about when and by whom the message was created. */ + metadata?: AgentMessageMetadata; +} +``` + +### Supporting Types + +```typescript +/** Reasoning information for an agent message. */ +interface AgentMessageReasoning { + /** When reasoning started. */ + start_date: string; + /** When reasoning ended. */ + end_date?: string; + /** Reasoning content. */ + content: string; +} + +/** A tool call made by the agent. */ +interface AgentMessageToolCall { + /** Tool call ID. */ + id: string; + /** Name of the tool called. */ + name: string; + /** Arguments passed to the tool as JSON string. */ + arguments_string: string; + /** Status of the tool call. */ + status: "running" | "success" | "error" | "stopped"; + /** Results from the tool call. */ + results?: string; +} + +/** Token usage statistics for an agent message. */ +interface AgentMessageUsage { + /** Number of tokens in the prompt. */ + prompt_tokens?: number; + /** Number of tokens in the completion. */ + completion_tokens?: number; +} + +/** Custom context provided with an agent message. */ +interface AgentMessageCustomContext { + /** Context message. */ + message: string; + /** Associated data for the context. */ + data: Record; + /** Type of context. */ + type: string; +} + +/** Metadata about when and by whom a message was created. */ +interface AgentMessageMetadata { + /** When the message was created. */ + created_date: string; + /** Email of the user who created the message. */ + created_by_email: string; + /** Full name of the user who created the message. */ + created_by_full_name: string; +} +``` + +### CreateConversationParams + +```typescript +/** Parameters for creating a new conversation. */ +interface CreateConversationParams { + /** The name of the agent to create a conversation with. */ + agent_name: string; + /** Optional metadata to attach to the conversation. */ + metadata?: Record; +} +``` + +### ModelFilterParams + +```typescript +/** Parameters for filtering, sorting, and paginating conversations. */ +interface ModelFilterParams { + /** Query object with field-value pairs for filtering. */ + q?: Record; + /** Sort parameter (e.g., "-created_date" for descending). */ + sort?: string | null; + /** Maximum number of results to return. */ + limit?: number | null; + /** Number of results to skip for pagination. */ + skip?: number | null; + /** Array of field names to include in the response. */ + fields?: string[] | null; +} +``` + +### AgentsModule + +```typescript +/** Agents module for managing AI agent conversations. */ +interface AgentsModule { + /** Gets all conversations from all agents in the app. */ + getConversations(): Promise; + + /** Gets a specific conversation by ID. */ + getConversation(conversationId: string): Promise; + + /** Lists conversations with filtering, sorting, and pagination. */ + listConversations(filterParams: ModelFilterParams): Promise; + + /** Creates a new conversation with an agent. */ + createConversation(conversation: CreateConversationParams): Promise; + + /** Adds a message to a conversation. */ + addMessage(conversation: AgentConversation, message: Partial): Promise; + + /** Subscribes to real-time updates for a conversation. Returns unsubscribe function. */ + subscribeToConversation(conversationId: string, onUpdate?: (conversation: AgentConversation) => void): () => void; + + /** Gets WhatsApp connection URL for an agent. */ + getWhatsAppConnectURL(agentName: string): string; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/client.md b/evals/fixtures/skill-check/skills/base44-sdk/references/client.md new file mode 100644 index 0000000..554bf2b --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/client.md @@ -0,0 +1,257 @@ +# Client Setup + +How to create and configure the Base44 client. + +## Contents +- [In Base44-Generated Apps](#in-base44-generated-apps) +- [In External Apps](#in-external-apps) +- [In Backend Functions](#in-backend-functions) +- [Authentication Modes](#authentication-modes) (Anonymous, User, Service Role) +- [Available Modules](#available-modules) +- [Client Methods](#client-methods) +- [Client Configuration Options](#client-configuration-options) + +## In Base44-Generated Apps + +The client is pre-configured and available as `base44`. Just use it: + +```javascript +const tasks = await base44.entities.Task.list(); +``` + +## In External Apps + +Install the SDK and create a client: + +```bash +npm install @base44/sdk +``` + +```javascript +import { createClient } from "@base44/sdk"; + +// IMPORTANT: The parameter name is 'appId' (NOT 'clientId', NOT 'id') +// IMPORTANT: onError must be nested inside 'options' object +const base44 = createClient({ + appId: "your-app-id", // Required: Use 'appId' parameter + token: "optional-user-token", // Optional: for pre-authenticated requests + options: { // Optional: configuration options + onError: (error) => { // Optional: error handler (must be in options) + console.error("Base44 error:", error); + } + } +}); +``` + +**Common Mistakes:** +- ❌ `createClient({ clientId: "..." })` - WRONG parameter name +- ❌ `createClient({ id: "..." })` - WRONG parameter name +- ❌ `createClient({ appId: "...", onError: ... })` - WRONG: onError must be in options +- ✅ `createClient({ appId: "..." })` - CORRECT parameter name +- ✅ `createClient({ appId: "...", options: { onError: ... } })` - CORRECT: onError in options + +## In Backend Functions + +Use `createClientFromRequest` to get a client with the caller's auth context: + +```javascript +import { createClientFromRequest } from "@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Client inherits authentication from the request + const user = await base44.auth.me(); + + return Response.json({ user }); +}); +``` + +## Authentication Modes + +| Mode | How to Get | Permissions | +|------|-----------|-------------| +| **Anonymous** | `createClient({ appId })` without token | Public data only | +| **User** | After `loginViaEmailPassword()` or via `createClientFromRequest` | User's own data | +| **Service Role** | `base44.asServiceRole.*` in backend | Full admin access | + +## Anonymous Mode + +No authentication. Can only access public resources. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Only works if Task entity allows anonymous read +const publicTasks = await base44.entities.Task.list(); +``` + +## User Mode + +After user logs in, the client automatically includes their token. + +```javascript +const base44 = createClient({ appId: "your-app-id" }); + +// Login sets the token +await base44.auth.loginViaEmailPassword("user@example.com", "password"); + +// Subsequent requests are authenticated +const user = await base44.auth.me(); +const myTasks = await base44.entities.Task.list(); // filtered by permissions +``` + +## Service Role Mode + +Admin-level access. **Backend only.** + +```javascript +// Inside a backend function +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // User mode - respects permissions + const myTasks = await base44.entities.Task.list(); + + // Service role - bypasses permissions + const allTasks = await base44.asServiceRole.entities.Task.list(); + const allUsers = await base44.asServiceRole.entities.User.list(); + const oauthToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + return Response.json({ myTasks, allTasks }); +}); +``` + +## Available Modules + +The client exposes these modules: + +```javascript +base44.entities // CRUD operations +base44.auth // Authentication +base44.agents // AI conversations +base44.functions // Backend function invocation +base44.integrations // Third-party services +base44.analytics // Event tracking +base44.appLogs // App usage logging +base44.users // User invitations + +// Service role only (backend) +base44.asServiceRole.entities +base44.asServiceRole.agents +base44.asServiceRole.functions +base44.asServiceRole.integrations +base44.asServiceRole.appLogs +base44.asServiceRole.connectors +``` + +## Client Methods + +The client provides these methods: + +```javascript +// Set authentication token for all subsequent requests +base44.setToken(newToken); + +// Cleanup WebSocket connections (call when done with client) +base44.cleanup(); +``` + +### setToken + +Updates the authentication token for all subsequent API requests and WebSocket connections. + +```javascript +// After receiving a token (e.g., from external auth) +base44.setToken("new-jwt-token"); +``` + +### cleanup + +Disconnects WebSocket connections. Call when you're done with the client or when the component unmounts. + +```javascript +// Cleanup on component unmount (React example) +useEffect(() => { + return () => base44.cleanup(); +}, []); +``` + +## Client Configuration Options + +```javascript +createClient({ + appId: "your-app-id", // Required: MUST use 'appId' (not 'clientId' or 'id') + token: "jwt-token", // Optional: pre-set auth token + options: { // Optional: configuration options + onError: (error) => {} // Optional: global error handler (must be in options) + } +}); +``` + +**⚠️ Critical:** +- The parameter name is `appId`, not `clientId` or `id`. Using the wrong parameter name will cause errors. +- The `onError` handler must be nested inside the `options` object, not at the top level. + +## Type Definitions + +### CreateClientConfig + +```typescript +/** Configuration for creating a Base44 client. */ +interface CreateClientConfig { + /** The Base44 app ID (required). */ + appId: string; + /** User authentication token. Used to authenticate as a specific user. */ + token?: string; + /** Service role authentication token (backend only). */ + serviceToken?: string; + /** Additional client options. */ + options?: CreateClientOptions; +} + +/** Options for creating a Base44 client. */ +interface CreateClientOptions { + /** Optional error handler called whenever an API error occurs. */ + onError?: (error: Error) => void; +} +``` + +### Base44Client + +```typescript +/** The Base44 client instance. */ +interface Base44Client { + /** Entities module for CRUD operations on your data models. */ + entities: EntitiesModule; + /** Integrations module for calling pre-built integration endpoints. */ + integrations: IntegrationsModule; + /** Auth module for user authentication and management. */ + auth: AuthModule; + /** Functions module for invoking custom backend functions. */ + functions: FunctionsModule; + /** Agents module for managing AI agent conversations. */ + agents: AgentsModule; + /** App logs module for tracking app usage. */ + appLogs: AppLogsModule; + /** Analytics module for tracking custom events. */ + analytics: AnalyticsModule; + + /** Cleanup function to disconnect WebSocket connections. */ + cleanup(): void; + + /** Sets a new authentication token for all subsequent requests. */ + setToken(newToken: string): void; + + /** Provides access to modules with elevated service role permissions (backend only). */ + readonly asServiceRole: { + entities: EntitiesModule; + integrations: IntegrationsModule; + connectors: ConnectorsModule; + functions: FunctionsModule; + agents: AgentsModule; + appLogs: AppLogsModule; + cleanup(): void; + }; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/connectors.md b/evals/fixtures/skill-check/skills/base44-sdk/references/connectors.md new file mode 100644 index 0000000..f3fbcb4 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/connectors.md @@ -0,0 +1,113 @@ +# Connectors Module + +OAuth token management for external services. **Service role only** (backend functions). + +## Method + +```javascript +base44.asServiceRole.connectors.getAccessToken(integrationType): Promise +``` + +Returns a raw OAuth access token for the specified service. + +## Supported Services + +| Integration Type | Service | +|-----------------|---------| +| `"googlecalendar"` | Google Calendar | +| `"googledrive"` | Google Drive | +| `"slack"` | Slack | +| `"notion"` | Notion | +| `"salesforce"` | Salesforce | +| `"hubspot"` | HubSpot | +| `"linkedin"` | LinkedIn | +| `"tiktok"` | TikTok | +| `"github"` | GitHub | + +## Example Usage + +```javascript +// Backend function only +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Get OAuth token for Slack + const slackToken = await base44.asServiceRole.connectors.getAccessToken("slack"); + + // Use token directly with Slack API + const response = await fetch("https://slack.com/api/chat.postMessage", { + method: "POST", + headers: { + "Authorization": `Bearer ${slackToken}`, + "Content-Type": "application/json" + }, + body: JSON.stringify({ + channel: "#general", + text: "Hello from Base44!" + }) + }); + + return Response.json(await response.json()); +}); +``` + +## Google Calendar Example + +```javascript +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + const token = await base44.asServiceRole.connectors.getAccessToken("googlecalendar"); + + // List upcoming events + const response = await fetch( + "https://www.googleapis.com/calendar/v3/calendars/primary/events?" + + new URLSearchParams({ + maxResults: "10", + orderBy: "startTime", + singleEvents: "true", + timeMin: new Date().toISOString() + }), + { + headers: { "Authorization": `Bearer ${token}` } + } + ); + + const events = await response.json(); + return Response.json(events); +}); +``` + +## Setup Requirements + +1. **Builder plan** or higher +2. **Backend functions** enabled +3. **Connector configured** in Base44 dashboard (OAuth flow completed) + +## Important Notes + +- **One account per connector per app**: All users share the same connected account +- **Backend only**: `connectors` module not available in frontend code +- **Service role required**: Must use `base44.asServiceRole.connectors` +- **You handle the API calls**: Base44 only provides the token; you make the actual API requests +- **Token refresh**: Base44 handles token refresh automatically + +## Type Definitions + +```typescript +/** + * The type of external integration/connector. + * Examples: 'googlecalendar', 'slack', 'github', 'notion', etc. + */ +type ConnectorIntegrationType = string; + +/** Connectors module for managing OAuth tokens for external services. */ +interface ConnectorsModule { + /** + * Retrieves an OAuth access token for a specific external integration type. + * @param integrationType - The type of integration (e.g., 'googlecalendar', 'slack'). + * @returns Promise resolving to the access token string. + */ + getAccessToken(integrationType: ConnectorIntegrationType): Promise; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/entities.md b/evals/fixtures/skill-check/skills/base44-sdk/references/entities.md new file mode 100644 index 0000000..a7bdf35 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/entities.md @@ -0,0 +1,253 @@ +# Entities Module + +CRUD operations on data models. Access via `base44.entities.EntityName.method()`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Create, Bulk Create, List, Filter, Get, Update, Delete, Subscribe) +- [User Entity](#user-entity) +- [Service Role Access](#service-role-access) +- [Permissions](#permissions) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `create(data)` | `Promise` | Create one record | +| `bulkCreate(dataArray)` | `Promise` | Create multiple records | +| `list(sort?, limit?, skip?, fields?)` | `Promise` | Get all records (paginated) | +| `filter(query, sort?, limit?, skip?, fields?)` | `Promise` | Get records matching conditions | +| `get(id)` | `Promise` | Get single record by ID | +| `update(id, data)` | `Promise` | Update record (partial update) | +| `delete(id)` | `Promise` | Delete record by ID | +| `deleteMany(query)` | `Promise` | Delete all matching records | +| `importEntities(file)` | `Promise` | Import from CSV (frontend only) | +| `subscribe(callback)` | `() => void` | Subscribe to realtime updates (returns unsubscribe function) | + +## Examples + +### Create + +```javascript +const task = await base44.entities.Task.create({ + title: "Complete documentation", + status: "pending", + dueDate: "2024-12-31" +}); +``` + +### Bulk Create + +```javascript +const tasks = await base44.entities.Task.bulkCreate([ + { title: "Task 1", status: "pending" }, + { title: "Task 2", status: "pending" } +]); +``` + +### List with Pagination + +```javascript +// Get first 10 records, sorted by created_date descending +const tasks = await base44.entities.Task.list( + "-created_date", // sort (string: prefix with - for descending) + 10, // limit + 0 // skip +); + +// Get next page +const page2 = await base44.entities.Task.list("-created_date", 10, 10); +``` + +### Filter + +```javascript +// Simple filter +const pending = await base44.entities.Task.filter({ status: "pending" }); + +// Multiple conditions +const myPending = await base44.entities.Task.filter({ + status: "pending", + assignedTo: userId +}); + +// With sort, limit, skip +const recent = await base44.entities.Task.filter( + { status: "pending" }, + "-created_date", // sort (string: prefix with - for descending) + 5, + 0 +); + +// Select specific fields +const titles = await base44.entities.Task.filter( + { status: "pending" }, + null, + null, + null, + ["id", "title"] +); +``` + +### Get by ID + +```javascript +const task = await base44.entities.Task.get("task-id-123"); +``` + +### Update + +```javascript +// Partial update - only specified fields change +await base44.entities.Task.update("task-id-123", { + status: "completed", + completedAt: new Date().toISOString() +}); +``` + +### Delete + +```javascript +// Single record +await base44.entities.Task.delete("task-id-123"); + +// Multiple records matching query +await base44.entities.Task.deleteMany({ status: "archived" }); +``` + +### Subscribe to Realtime Updates + +```javascript +// Subscribe to all changes on Task entity +const unsubscribe = base44.entities.Task.subscribe((event) => { + console.log(`Task ${event.id} was ${event.type}:`, event.data); + // event.type is "create", "update", or "delete" +}); + +// Later: unsubscribe to stop receiving updates +unsubscribe(); +``` + +**Event structure:** +```javascript +{ + type: "create" | "update" | "delete", + data: { ... }, // the entity data + id: "entity-id", // the affected entity's ID + timestamp: "2024-01-15T10:30:00Z" +} +``` + +## User Entity + +Every app has a built-in `User` entity with special rules: + +- Regular users can only read/update **their own** record +- Cannot create users via `entities.create()` - use `auth.register()` instead +- Service role has full access to all user records + +```javascript +// Get current user's record +const me = await base44.entities.User.get(currentUserId); + +// Service role: get any user +const anyUser = await base44.asServiceRole.entities.User.get(userId); +``` + +## Service Role Access + +For admin-level operations (bypass user permissions): + +```javascript +// Backend only +const allTasks = await base44.asServiceRole.entities.Task.list(); +const allUsers = await base44.asServiceRole.entities.User.list(); +``` + +## Permissions (RLS & FLS) + +Data access is controlled by **Row Level Security (RLS)** and **Field Level Security (FLS)** rules defined in entity schemas. + +1. **Authentication level**: anonymous, authenticated, or service role +2. **RLS rules**: Control which records (rows) users can create/read/update/delete +3. **FLS rules**: Control which fields users can read/write within accessible records + +Operations succeed or fail based on these rules - no partial results. + +RLS and FLS are configured in entity schema files (`base44/entities/*.jsonc`). See [entities-create.md](../../base44-cli/references/entities-create.md#row-level-security-rls) for configuration details. + +**Note:** `asServiceRole` sets the user's role to `"admin"` but does NOT bypass RLS. Your RLS rules must include admin access (e.g., `{ "user_condition": { "role": "admin" } }`) for service role operations to succeed. + +## Type Definitions + +### RealtimeEvent + +```typescript +/** Event types for realtime entity updates. */ +type RealtimeEventType = "create" | "update" | "delete"; + +/** Payload received when a realtime event occurs. */ +interface RealtimeEvent { + /** The type of change that occurred. */ + type: RealtimeEventType; + /** The entity data. */ + data: any; + /** The unique identifier of the affected entity. */ + id: string; + /** ISO 8601 timestamp of when the event occurred. */ + timestamp: string; +} + +/** Callback function invoked when a realtime event occurs. */ +type RealtimeCallback = (event: RealtimeEvent) => void; + +/** Function returned from subscribe, call it to unsubscribe. */ +type Subscription = () => void; +``` + +### EntityHandler + +```typescript +/** Entity handler providing CRUD operations for a specific entity type. */ +interface EntityHandler { + /** Lists records with optional pagination and sorting. */ + list(sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Filters records based on a query. */ + filter(query: Record, sort?: string, limit?: number, skip?: number, fields?: string[]): Promise; + + /** Gets a single record by ID. */ + get(id: string): Promise; + + /** Creates a new record. */ + create(data: Record): Promise; + + /** Updates an existing record. */ + update(id: string, data: Record): Promise; + + /** Deletes a single record by ID. */ + delete(id: string): Promise; + + /** Deletes multiple records matching a query. */ + deleteMany(query: Record): Promise; + + /** Creates multiple records in a single request. */ + bulkCreate(data: Record[]): Promise; + + /** Imports records from a file (frontend only). */ + importEntities(file: File): Promise; + + /** Subscribes to realtime updates. Returns unsubscribe function. */ + subscribe(callback: RealtimeCallback): Subscription; +} +``` + +### EntitiesModule + +```typescript +/** Entities module for managing app data. */ +interface EntitiesModule { + /** Access any entity by name dynamically. */ + [entityName: string]: EntityHandler; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/functions.md b/evals/fixtures/skill-check/skills/base44-sdk/references/functions.md new file mode 100644 index 0000000..8072465 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/functions.md @@ -0,0 +1,216 @@ +# Functions Module + +Invoke custom backend functions via `base44.functions`. + +## Contents +- [Method](#method) +- [Invoking Functions](#invoking-functions) (Frontend, File Upload, Service Role, REST API) +- [Writing Backend Functions](#writing-backend-functions) (Basic, Service Role, Secrets, Errors) +- [Setup Requirements](#setup-requirements) +- [Authentication Modes](#authentication-modes) + +## Method + +```javascript +base44.functions.invoke(functionName, data): Promise +``` + +- `functionName`: Name of the backend function +- `data`: Object of parameters (sent as JSON, or multipart if contains File objects) +- Returns: Whatever the function returns + +## Invoking Functions + +### From Frontend + +```javascript +const result = await base44.functions.invoke("processOrder", { + orderId: "order-123", + action: "ship" +}); + +console.log(result); +``` + +### With File Upload + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const file = fileInput.files[0]; + +// Automatically uses multipart/form-data when File objects present +const result = await base44.functions.invoke("uploadDocument", { + file: file, + category: "invoices" +}); +``` + +### With Service Role (Backend) + +```javascript +// Inside another backend function +const result = await base44.asServiceRole.functions.invoke("adminTask", { + userId: "user-123" +}); +``` + +### Via REST API (curl) + +Functions can be called via HTTP POST to your app domain: + +```bash +curl -X POST "https:///functions/" \ + -H "Content-Type: application/json" \ + -d '{"key": "value"}' +``` + +## Writing Backend Functions + +Backend functions run on Deno. Must export using `Deno.serve()`. + +### Required Directory Structure + +Each function must be in its own subdirectory under `base44/functions/` with a configuration file: + +``` +base44/ + functions/ + process-order/ # kebab-case directory name + function.jsonc # required configuration + index.ts # entry point +``` + +**function.jsonc:** +```jsonc +{ + "name": "process-order", + "entry": "index.ts" +} +``` + +For complete setup and deployment instructions, see [functions-create.md](../../base44-cli/references/functions-create.md) in base44-cli. + +### Basic Structure + +```javascript +// base44/functions/process-order/index.ts +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + // Get authenticated client from request + const base44 = createClientFromRequest(req); + + // Parse input + const { orderId, action } = await req.json(); + + // Your logic here + const order = await base44.entities.Orders.get(orderId); + + // Return response + return Response.json({ + success: true, + order: order + }); +}); +``` + +### With Service Role Access + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + const base44 = createClientFromRequest(req); + + // Check user is authenticated + const user = await base44.auth.me(); + if (!user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + + // Use service role for admin operations + const allOrders = await base44.asServiceRole.entities.Orders.list(); + + return Response.json({ orders: allOrders }); +}); +``` + +### Using Secrets + +```javascript +Deno.serve(async (req) => { + // Access environment variables (configured in app settings) + const apiKey = Deno.env.get("STRIPE_API_KEY"); + + const response = await fetch("https://api.stripe.com/v1/charges", { + headers: { + "Authorization": `Bearer ${apiKey}` + } + }); + + return Response.json(await response.json()); +}); +``` + +### Error Handling + +```javascript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req) => { + try { + const base44 = createClientFromRequest(req); + const { orderId } = await req.json(); + + const order = await base44.entities.Orders.get(orderId); + if (!order) { + return Response.json( + { error: "Order not found" }, + { status: 404 } + ); + } + + return Response.json({ order }); + + } catch (error) { + return Response.json( + { error: error.message }, + { status: 500 } + ); + } +}); +``` + +## Setup Requirements + +1. Enable Backend Functions in app settings (requires appropriate plan) +2. Create function files in `/functions` folder +3. Configure secrets via app dashboard for API keys + +## Authentication Modes + +| Mode | Context | Permissions | +|------|---------|-------------| +| User | `base44.functions.invoke()` | Runs under calling user's permissions | +| Service Role | `base44.asServiceRole.functions.invoke()` | Admin-level access | + +Inside the function, use `createClientFromRequest(req)` to get a client that inherits the caller's auth context. + +## Type Definitions + +```typescript +/** Functions module for invoking custom backend functions. */ +interface FunctionsModule { + /** + * Invokes a custom backend function by name. + * + * If any parameter is a File object, the request will automatically be + * sent as multipart/form-data. Otherwise, it will be sent as JSON. + * + * @param functionName - The name of the function to invoke. + * @param data - An object containing named parameters for the function. + * @returns Promise resolving to the function's response. + */ + invoke(functionName: string, data: Record): Promise; +} +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/integrations.md b/evals/fixtures/skill-check/skills/base44-sdk/references/integrations.md new file mode 100644 index 0000000..f356a18 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/integrations.md @@ -0,0 +1,377 @@ +# Integrations Module + +Access third-party services via `base44.integrations`. + +## Types of Integrations + +1. **Core/Built-in**: AI, email, file uploads (available by default) +2. **Catalog integrations**: Pre-built connectors from Base44 catalog +3. **Custom integrations**: Your own OpenAPI-based integrations + +## Accessing Integrations + +```javascript +// Core integrations +base44.integrations.Core.FunctionName(params) + +// Custom integrations +base44.integrations.custom.call(slug, operationId, params) +``` + +## Core Integrations + +### InvokeLLM (AI Text Generation) + +Generate text or structured JSON data using AI models. + +```javascript +// Basic prompt - returns string +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Summarize this text: ..." +}); + +// With internet context (uses Google Search, Maps, News) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "What's the current weather in NYC?", + add_context_from_internet: true +}); + +// Structured JSON response - returns object +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Analyze the sentiment of: 'Great product but slow shipping'", + response_json_schema: { + type: "object", + properties: { + sentiment: { type: "string", enum: ["positive", "negative", "mixed"] }, + score: { type: "number", description: "Score from 1-10" }, + key_points: { type: "array", items: { type: "string" } } + } + } +}); +// Returns: { sentiment: "mixed", score: 7, key_points: ["great product", "slow shipping"] } + +// With file attachments (uploaded via UploadFile) +const response = await base44.integrations.Core.InvokeLLM({ + prompt: "Describe what's in this image", + file_urls: ["https://...uploaded_image.png"] +}); +``` + +**Parameters:** +- `prompt` (string, required): The prompt text to send to the model +- `add_context_from_internet` (boolean, optional): If true, uses Google Search/Maps/News for real-time context +- `response_json_schema` (object, optional): JSON schema for structured output +- `file_urls` (string[], optional): URLs of uploaded files for context + +### GenerateImage + +Create AI-generated images from text prompts. + +```javascript +const { url } = await base44.integrations.Core.GenerateImage({ + prompt: "A serene mountain landscape with a lake" +}); +console.log(url); // https://...generated_image.png +``` + +### SendEmail + +Send emails to registered users. Every app gets this integration (no plan upgrade required). + +```javascript +await base44.integrations.Core.SendEmail({ + to: "user@example.com", + subject: "Welcome!", + body: "

Hello

Welcome to our app.

", + from_name: "My App" // optional, defaults to app name +}); +``` + +**Parameters:** +- `to` (string, required): Recipient email address +- `subject` (string, required): Email subject line +- `body` (string, required): Plain text or HTML email body +- `from_name` (string, optional): Sender name displayed to recipient + +**Limitations:** +- 1 credit per email (2 credits with custom domain) + +### UploadFile (Public) + +Upload files to public storage. + +```javascript +const fileInput = document.querySelector('input[type="file"]'); +const { file_url } = await base44.integrations.Core.UploadFile({ + file: fileInput.files[0] +}); +console.log(file_url); // https://...uploaded_file.pdf +``` + +### UploadPrivateFile + +Upload files to private storage that requires a signed URL to access. + +```javascript +const { file_uri } = await base44.integrations.Core.UploadPrivateFile({ + file: fileInput.files[0] +}); +console.log(file_uri); // "private/user123/document.pdf" + +// Create a signed URL to access the file +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri, + expires_in: 3600 // URL expires in 1 hour (default: 300 seconds) +}); +console.log(signed_url); // Temporary URL to access the private file +``` + +### CreateFileSignedUrl + +Generate temporary access links for private files. + +```javascript +const { signed_url } = await base44.integrations.Core.CreateFileSignedUrl({ + file_uri: "private/user123/document.pdf", + expires_in: 7200 // 2 hours +}); +``` + +**Parameters:** +- `file_uri` (string, required): URI from UploadPrivateFile +- `expires_in` (number, optional): Expiration time in seconds (default: 300) + +### ExtractDataFromUploadedFile + +Extract structured data from uploaded files using AI. + +```javascript +// First upload the file +const { file_url } = await base44.integrations.Core.UploadFile({ file }); + +// Then extract structured data +const result = await base44.integrations.Core.ExtractDataFromUploadedFile({ + file_url, + json_schema: { + type: "object", + properties: { + invoice_number: { type: "string" }, + total_amount: { type: "number" }, + date: { type: "string" }, + vendor_name: { type: "string" } + } + } +}); +console.log(result); // { invoice_number: "INV-12345", total_amount: 1250.00, ... } +``` + +## Custom Integrations + +Custom integrations allow workspace administrators to connect any external API by importing an OpenAPI specification. Use `base44.integrations.custom.call()` to invoke them. + +### Syntax + +```javascript +const response = await base44.integrations.custom.call( + slug, // Integration identifier (set by admin) + operationId, // Endpoint in "method:path" format + params // Optional: payload, pathParams, queryParams +); +``` + +### Examples + +```javascript +// GET request with query params +const response = await base44.integrations.custom.call( + "my-crm", + "get:/contacts", + { queryParams: { limit: 10, status: "active" } } +); + +// POST request with body +const response = await base44.integrations.custom.call( + "my-crm", + "post:/contacts", + { payload: { name: "John Doe", email: "john@example.com" } } +); + +// Request with path parameters +const response = await base44.integrations.custom.call( + "github", + "post:/repos/{owner}/{repo}/issues", + { + pathParams: { owner: "myorg", repo: "myrepo" }, + payload: { title: "Bug report", body: "Something is broken" } + } +); +``` + +### Response Structure + +```javascript +{ + success: true, // Whether external API returned 2xx + status_code: 200, // HTTP status code + data: { ... } // Response from external API +} +``` + +## Requirements + +- **Core integrations**: Available on all plans +- **Catalog/Custom integrations**: Require Builder plan or higher + +## Type Definitions + +### Core Integration Parameters + +```typescript +/** Parameters for the InvokeLLM function. */ +interface InvokeLLMParams { + /** The prompt text to send to the model. */ + prompt: string; + /** If true, uses Google Search/Maps/News for real-time context. */ + add_context_from_internet?: boolean; + /** JSON schema for structured output. If provided, returns object instead of string. */ + response_json_schema?: object; + /** File URLs (from UploadFile) to provide as context. */ + file_urls?: string[]; +} + +/** Parameters for the GenerateImage function. */ +interface GenerateImageParams { + /** Description of the image to generate. */ + prompt: string; +} + +/** Result from GenerateImage. */ +interface GenerateImageResult { + /** URL of the generated image. */ + url: string; +} + +/** Parameters for the UploadFile function. */ +interface UploadFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadFile. */ +interface UploadFileResult { + /** URL of the uploaded file. */ + file_url: string; +} + +/** Parameters for the SendEmail function. */ +interface SendEmailParams { + /** Recipient email address. */ + to: string; + /** Email subject line. */ + subject: string; + /** Plain text or HTML email body. */ + body: string; + /** Sender name (defaults to app name). */ + from_name?: string; +} + +/** Parameters for ExtractDataFromUploadedFile. */ +interface ExtractDataFromUploadedFileParams { + /** URL of the uploaded file. */ + file_url: string; + /** JSON schema defining fields to extract. */ + json_schema: object; +} + +/** Parameters for UploadPrivateFile. */ +interface UploadPrivateFileParams { + /** The file object to upload. */ + file: File; +} + +/** Result from UploadPrivateFile. */ +interface UploadPrivateFileResult { + /** URI of the private file (used for signed URLs). */ + file_uri: string; +} + +/** Parameters for CreateFileSignedUrl. */ +interface CreateFileSignedUrlParams { + /** URI from UploadPrivateFile. */ + file_uri: string; + /** Expiration time in seconds (default: 300). */ + expires_in?: number; +} + +/** Result from CreateFileSignedUrl. */ +interface CreateFileSignedUrlResult { + /** Temporary signed URL to access the file. */ + signed_url: string; +} +``` + +### CoreIntegrations + +```typescript +/** Core package containing built-in Base44 integration functions. */ +interface CoreIntegrations { + InvokeLLM(params: InvokeLLMParams): Promise; + GenerateImage(params: GenerateImageParams): Promise; + UploadFile(params: UploadFileParams): Promise; + SendEmail(params: SendEmailParams): Promise; + ExtractDataFromUploadedFile(params: ExtractDataFromUploadedFileParams): Promise; + UploadPrivateFile(params: UploadPrivateFileParams): Promise; + CreateFileSignedUrl(params: CreateFileSignedUrlParams): Promise; +} +``` + +### Custom Integrations + +```typescript +/** Parameters for calling a custom integration endpoint. */ +interface CustomIntegrationCallParams { + /** Request body payload. */ + payload?: Record; + /** Path parameters to substitute in the URL. */ + pathParams?: Record; + /** Query string parameters. */ + queryParams?: Record; + /** Additional headers for this request. */ + headers?: Record; +} + +/** Response from a custom integration call. */ +interface CustomIntegrationCallResponse { + /** Whether the external API returned a 2xx status code. */ + success: boolean; + /** The HTTP status code from the external API. */ + status_code: number; + /** The response data from the external API. */ + data: any; +} + +/** Module for calling custom workspace-level API integrations. */ +interface CustomIntegrationsModule { + /** + * Call a custom integration endpoint. + * @param slug - The integration's unique identifier. + * @param operationId - The operation ID from the OpenAPI spec. + * @param params - Optional payload, pathParams, queryParams, headers. + */ + call(slug: string, operationId: string, params?: CustomIntegrationCallParams): Promise; +} +``` + +### IntegrationsModule + +```typescript +/** Integrations module for calling integration endpoints. */ +type IntegrationsModule = { + /** Core package with built-in integrations. */ + Core: CoreIntegrations; + /** Custom integrations module. */ + custom: CustomIntegrationsModule; + /** Additional integration packages (dynamic). */ + [packageName: string]: any; +}; +``` diff --git a/evals/fixtures/skill-check/skills/base44-sdk/references/users.md b/evals/fixtures/skill-check/skills/base44-sdk/references/users.md new file mode 100644 index 0000000..4205b06 --- /dev/null +++ b/evals/fixtures/skill-check/skills/base44-sdk/references/users.md @@ -0,0 +1,82 @@ +# Users Module + +Invite users to the app via `base44.users`. + +## Contents +- [Methods](#methods) +- [Examples](#examples) (Invite User) +- [Roles](#roles) +- [Notes](#notes) + +## Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `inviteUser(user_email, role)` | `Promise` | Invite a user to the app | + +## Examples + +### Invite User + +```javascript +// Invite a user with "user" role +await base44.users.inviteUser("newuser@example.com", "user"); + +// Invite an admin +await base44.users.inviteUser("admin@example.com", "admin"); +``` + +### Invite Multiple Users + +```javascript +const usersToInvite = [ + { email: "user1@example.com", role: "user" }, + { email: "user2@example.com", role: "user" }, + { email: "manager@example.com", role: "admin" } +]; + +for (const user of usersToInvite) { + await base44.users.inviteUser(user.email, user.role); + console.log(`Invited ${user.email} as ${user.role}`); +} +``` + +## Roles + +The `role` parameter must be one of: + +| Role | Description | +|------|-------------| +| `"user"` | Standard user with default permissions | +| `"admin"` | Administrator with elevated permissions | + +**Note:** Only `"user"` and `"admin"` are valid role values. An error will be thrown if you pass any other value. + +## Notes + +- **Email invitation**: The invited user receives an email with a link to join the app +- **Duplicate handling**: Inviting an existing user will re-send the invitation +- **Also available in auth**: `base44.auth.inviteUser()` provides the same functionality +- **Role validation**: Only `"user"` or `"admin"` are accepted + +```javascript +// These are equivalent: +await base44.users.inviteUser("newuser@example.com", "user"); +await base44.auth.inviteUser("newuser@example.com", "user"); +``` + +## Type Definitions + +```typescript +/** Users module for inviting users to the app. */ +interface UsersModule { + /** + * Invite a user to the application. + * @param user_email - User's email address. + * @param role - User's role ('user' or 'admin'). + * @returns Promise resolving when the invitation is sent. + * @throws Error if role is not 'user' or 'admin'. + */ + inviteUser(user_email: string, role: "user" | "admin"): Promise; +} +``` diff --git a/evals/package-lock.json b/evals/package-lock.json new file mode 100644 index 0000000..eebd3cd --- /dev/null +++ b/evals/package-lock.json @@ -0,0 +1,891 @@ +{ + "name": "skills-evals", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "skills-evals", + "version": "1.0.0", + "dependencies": { + "execa": "^9.5.2", + "zod": "^3.24.2" + }, + "devDependencies": { + "@types/node": "^22.13.1", + "tsx": "^4.19.2", + "typescript": "^5.7.3" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", + "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz", + "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", + "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz", + "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", + "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", + "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", + "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", + "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", + "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", + "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", + "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", + "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", + "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", + "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", + "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", + "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", + "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", + "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", + "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", + "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", + "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz", + "integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", + "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", + "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", + "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", + "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@sec-ant/readable-stream": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@sec-ant/readable-stream/-/readable-stream-0.4.1.tgz", + "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", + "license": "MIT" + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@types/node": { + "version": "22.19.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", + "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/esbuild": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", + "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.2", + "@esbuild/android-arm": "0.27.2", + "@esbuild/android-arm64": "0.27.2", + "@esbuild/android-x64": "0.27.2", + "@esbuild/darwin-arm64": "0.27.2", + "@esbuild/darwin-x64": "0.27.2", + "@esbuild/freebsd-arm64": "0.27.2", + "@esbuild/freebsd-x64": "0.27.2", + "@esbuild/linux-arm": "0.27.2", + "@esbuild/linux-arm64": "0.27.2", + "@esbuild/linux-ia32": "0.27.2", + "@esbuild/linux-loong64": "0.27.2", + "@esbuild/linux-mips64el": "0.27.2", + "@esbuild/linux-ppc64": "0.27.2", + "@esbuild/linux-riscv64": "0.27.2", + "@esbuild/linux-s390x": "0.27.2", + "@esbuild/linux-x64": "0.27.2", + "@esbuild/netbsd-arm64": "0.27.2", + "@esbuild/netbsd-x64": "0.27.2", + "@esbuild/openbsd-arm64": "0.27.2", + "@esbuild/openbsd-x64": "0.27.2", + "@esbuild/openharmony-arm64": "0.27.2", + "@esbuild/sunos-x64": "0.27.2", + "@esbuild/win32-arm64": "0.27.2", + "@esbuild/win32-ia32": "0.27.2", + "@esbuild/win32-x64": "0.27.2" + } + }, + "node_modules/execa": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-9.6.1.tgz", + "integrity": "sha512-9Be3ZoN4LmYR90tUoVu2te2BsbzHfhJyfEiAVfz7N5/zv+jduIfLrV2xdQXOHbaD6KgpGdO9PRPM1Y4Q9QkPkA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "cross-spawn": "^7.0.6", + "figures": "^6.1.0", + "get-stream": "^9.0.0", + "human-signals": "^8.0.1", + "is-plain-obj": "^4.1.0", + "is-stream": "^4.0.1", + "npm-run-path": "^6.0.0", + "pretty-ms": "^9.2.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^4.0.0", + "yoctocolors": "^2.1.1" + }, + "engines": { + "node": "^18.19.0 || >=20.5.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/figures": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", + "integrity": "sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==", + "license": "MIT", + "dependencies": { + "is-unicode-supported": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-stream": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-9.0.1.tgz", + "integrity": "sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==", + "license": "MIT", + "dependencies": { + "@sec-ant/readable-stream": "^0.4.1", + "is-stream": "^4.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-tsconfig": { + "version": "4.13.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.1.tgz", + "integrity": "sha512-EoY1N2xCn44xU6750Sx7OjOIT59FkmstNc3X6y5xpz7D5cBtZRe/3pSlTkDJgqsOk3WwZPkWfonhhUJfttQo3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/human-signals": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-8.0.1.tgz", + "integrity": "sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-4.0.1.tgz", + "integrity": "sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", + "integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/npm-run-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-6.0.0.tgz", + "integrity": "sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==", + "license": "MIT", + "dependencies": { + "path-key": "^4.0.0", + "unicorn-magic": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse-ms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", + "integrity": "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-ms": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.0.tgz", + "integrity": "sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==", + "license": "MIT", + "dependencies": { + "parse-ms": "^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/strip-final-newline": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-4.0.0.tgz", + "integrity": "sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unicorn-magic": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz", + "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/yoctocolors": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors/-/yoctocolors-2.1.2.tgz", + "integrity": "sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/evals/package.json b/evals/package.json new file mode 100644 index 0000000..cffdf70 --- /dev/null +++ b/evals/package.json @@ -0,0 +1,24 @@ +{ + "name": "skills-evals", + "version": "1.0.0", + "description": "Evaluation system for skills testing", + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "setup": "tsx src/setup.ts", + "eval": "npm run setup && tsx src/index.ts", + "eval:only": "tsx src/index.ts", + "eval:compare": "tsx src/compare.ts", + "test": "npm run eval" + }, + "dependencies": { + "execa": "^9.5.2", + "zod": "^3.24.2" + }, + "devDependencies": { + "@types/node": "^22.13.1", + "tsx": "^4.19.2", + "typescript": "^5.7.3" + } +} diff --git a/evals/src/agents/claude-code.ts b/evals/src/agents/claude-code.ts new file mode 100644 index 0000000..e09df1f --- /dev/null +++ b/evals/src/agents/claude-code.ts @@ -0,0 +1,147 @@ +import { execSync } from 'child_process'; +import type { CodingAgent, CodingAgentResponse } from '../types.js'; + +// Skill markers for detecting which skills were invoked +const skillMarkers: Record = { + 'base44-cli': [ + // CLI commands + /base44\s+(entities|functions|agents)\s+(push|deploy|pull|create)/i, + /npx\s+base44/i, + /yarn\s+base44/i, + /pnpm\s+base44/i, + // File paths (more flexible patterns) + /base44\/entities\/[\w-]+\.jsonc/i, + /base44\/functions\/[\w-]+\//i, + /base44\/agents\/[\w_]+\.jsonc/i, + /`base44\/entities\//i, + /`base44\/functions\//i, + /`base44\/agents\//i, + // Config files + /function\.jsonc/i, + /\.jsonc.*entity|entity.*\.jsonc/i, + // Deno/function patterns + /Deno\.serve/i, + /createClientFromRequest/i, + /npm:@base44\/sdk/i, + // Generic mentions + /entities\s+push/i, + /functions\s+deploy/i, + /agents\s+push/i, + // Agent config patterns + /tool_configs/i, + /entity_name.*allowed_operations|allowed_operations.*entity_name/i, + /function_name.*description|"function_name"/i, + ], + 'base44-sdk': [ + // SDK imports + /@base44\/sdk/i, + /from\s+['"]@base44\/sdk['"]/i, + /import.*createClient.*from.*@base44\/sdk/i, + // Entity operations (flexible) + /base44\.entities\.\w+\.(create|list|filter|update|delete|get|subscribe)/i, + /entities\.[A-Z]\w*\.(create|list|filter|update|delete|get|subscribe)/i, + /\.entities\.\w+\.list\(\)/i, + /\.entities\.\w+\.create\(/i, + /\.entities\.\w+\.update\(/i, + // Integrations + /integrations\.Core\./i, + /SendEmail|InvokeLLM|UploadFile|GenerateImage/i, + // Auth + /base44\.auth\./i, + // Functions invoke + /base44\.functions\.invoke/i, + /functions\.invoke/i, + // Agent SDK methods + /base44\.agents\./i, + /\.agents\.createConversation/i, + /\.agents\.addMessage/i, + /\.agents\.subscribeToConversation/i, + /\.agents\.getConversation/i, + /createConversation.*agent_name|agent_name.*createConversation/i, + /addMessage.*role.*content|role.*user.*content/i, + /subscribeToConversation/i, + // Service role + /asServiceRole/i, + ], +}; + +function detectSkills(output: string): string[] { + const detectedSkills: Set = new Set(); + + for (const [skill, patterns] of Object.entries(skillMarkers)) { + for (const pattern of patterns) { + if (pattern.test(output)) { + detectedSkills.add(skill); + break; + } + } + } + + return Array.from(detectedSkills); +} + +export class ClaudeCodeAgent implements CodingAgent { + name = 'claude-code'; + verbose = false; + + async run(prompt: string, workingDir: string): Promise { + try { + if (this.verbose) { + console.log(`[ClaudeCode] Working dir: ${workingDir}`); + } + + // Escape the prompt for shell + const escapedPrompt = prompt.replace(/"/g, '\\"'); + const command = `claude -p "${escapedPrompt}"`; + + if (this.verbose) { + console.log(`[ClaudeCode] Running: ${command}`); + } + + const output = execSync(command, { + cwd: workingDir, + encoding: 'utf8', + timeout: 300000, // 5 minute timeout + maxBuffer: 10 * 1024 * 1024, // 10MB buffer + }); + + if (this.verbose) { + console.log(`[ClaudeCode] Output length: ${output.length}`); + console.log(`[ClaudeCode] Output preview: ${output.substring(0, 200)}`); + } + + const skillsInvoked = detectSkills(output); + + return { + output, + skillsInvoked, + metadata: { + exitCode: 0, + }, + }; + } catch (error: unknown) { + const execError = error as { stdout?: string; stderr?: string; message?: string }; + const errorMessage = execError.message || String(error); + const stdout = execError.stdout || ''; + const stderr = execError.stderr || ''; + + if (this.verbose) { + console.log(`[ClaudeCode] Error: ${errorMessage}`); + console.log(`[ClaudeCode] Stdout: ${stdout}`); + console.log(`[ClaudeCode] Stderr: ${stderr}`); + } + + // If we got output despite error, use it + const output = stdout || stderr || `Error running Claude Code: ${errorMessage}`; + const skillsInvoked = detectSkills(output); + + return { + output, + skillsInvoked, + metadata: { + error: errorMessage, + }, + }; + } + } +} diff --git a/evals/src/agents/interface.ts b/evals/src/agents/interface.ts new file mode 100644 index 0000000..2e119d8 --- /dev/null +++ b/evals/src/agents/interface.ts @@ -0,0 +1,27 @@ +import type { CodingAgent, CodingAgentResponse } from '../types.js'; + +export type { CodingAgent, CodingAgentResponse }; + +export function createAgent(name: string): CodingAgent { + switch (name) { + case 'mock': + return { + name: 'mock', + async run(prompt: string, workingDir: string): Promise { + const { MockAgent } = await import('./mock.js'); + const agent = new MockAgent(); + return agent.run(prompt, workingDir); + } + }; + case 'claude-code': + default: + return { + name: 'claude-code', + async run(prompt: string, workingDir: string): Promise { + const { ClaudeCodeAgent } = await import('./claude-code.js'); + const agent = new ClaudeCodeAgent(); + return agent.run(prompt, workingDir); + } + }; + } +} diff --git a/evals/src/agents/mock.ts b/evals/src/agents/mock.ts new file mode 100644 index 0000000..f7fea0d --- /dev/null +++ b/evals/src/agents/mock.ts @@ -0,0 +1,145 @@ +import type { CodingAgent, CodingAgentResponse } from '../types.js'; + +/** + * Mock agent for testing the eval infrastructure. + * Returns a predefined response based on the prompt. + */ +export class MockAgent implements CodingAgent { + name = 'mock'; + + async run(prompt: string, workingDir: string): Promise { + console.log(`[MockAgent] Running in: ${workingDir}`); + console.log(`[MockAgent] Prompt: ${prompt.substring(0, 100)}...`); + + // Simulate different responses based on prompt content + let output = ''; + const skillsInvoked: string[] = []; + + if (prompt.toLowerCase().includes('entity') || prompt.toLowerCase().includes('todo')) { + output = `I'll create the entity files for you. + +Here's the Todo entity in \`base44/entities/todo.jsonc\`: + +\`\`\`jsonc +{ + "name": "Todo", + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "The todo title" + }, + "completed": { + "type": "boolean", + "default": false + } + }, + "required": ["title"] +} +\`\`\` + +To push these entities to Base44, run: +\`\`\`bash +npx base44 entities push +\`\`\` +`; + skillsInvoked.push('base44-cli'); + } else if (prompt.toLowerCase().includes('function')) { + output = `I'll create the function for you. + +Creating \`base44/functions/send-order-notification/function.jsonc\`: + +\`\`\`jsonc +{ + "name": "send-order-notification", + "description": "Sends notification emails for order status changes" +} +\`\`\` + +And \`base44/functions/send-order-notification/index.ts\`: + +\`\`\`typescript +import { createClientFromRequest } from "npm:@base44/sdk"; + +Deno.serve(async (req: Request) => { + const base44 = createClientFromRequest(req); + const { order_id, new_status } = await req.json(); + + const order = await base44.asServiceRole.entities.Order.get(order_id); + const user = await base44.asServiceRole.entities.User.get(order.user_id); + + await base44.asServiceRole.integrations.Core.SendEmail({ + to: user.email, + subject: \`Order Status Update\`, + body: \`Your order status is now: \${new_status}\` + }); + + return Response.json({ success: true }); +}); +\`\`\` + +Deploy with: +\`\`\`bash +npx base44 functions deploy +\`\`\` +`; + skillsInvoked.push('base44-cli'); + } else if (prompt.toLowerCase().includes('tasklist') || prompt.toLowerCase().includes('sdk')) { + output = `I'll implement the TaskList component using the Base44 SDK. + +\`\`\`typescript +import { useState, useEffect } from 'react'; +import { base44 } from '../lib/base44'; + +interface Task { + id: string; + title: string; + status: string; + due_date?: string; +} + +export function TaskList() { + const [tasks, setTasks] = useState([]); + + useEffect(() => { + loadTasks(); + }, []); + + async function loadTasks() { + const data = await base44.entities.Task.list(); + setTasks(data); + } + + async function toggleComplete(task: Task) { + const newStatus = task.status === 'completed' ? 'pending' : 'completed'; + await base44.entities.Task.update(task.id, { status: newStatus }); + loadTasks(); + } + + return ( +
    + {tasks.map(task => ( +
  • toggleComplete(task)}> + {task.title} - {task.status} +
    • + ))} +
+ ); +} +\`\`\` +`; + skillsInvoked.push('base44-sdk'); + } else { + output = `I can help you with that. This is a Base44 project.`; + } + + return { + output, + skillsInvoked, + metadata: { + mock: true, + workingDir, + }, + }; + } +} diff --git a/evals/src/checks/agent-config.ts b/evals/src/checks/agent-config.ts new file mode 100644 index 0000000..85e9a14 --- /dev/null +++ b/evals/src/checks/agent-config.ts @@ -0,0 +1,105 @@ +import { z } from 'zod'; +import fs from 'fs/promises'; +import path from 'path'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +// Agent configuration schema based on base44-cli agent structure +const ToolConfigSchema = z.object({ + name: z.string(), + enabled: z.boolean().optional(), + config: z.record(z.any()).optional(), +}); + +const AgentConfigSchema = z.object({ + name: z.string().min(1), + description: z.string().optional(), + instructions: z.string().optional(), + model: z.string().optional(), + tool_configs: z.array(ToolConfigSchema).optional(), + temperature: z.number().min(0).max(2).optional(), + max_tokens: z.number().positive().optional(), +}); + +function extractJsonFromOutput(output: string): object | null { + // Try to find JSON in code blocks first + const codeBlockMatch = output.match(/```(?:json|jsonc)?\s*([\s\S]*?)```/); + if (codeBlockMatch) { + try { + const jsonStr = codeBlockMatch[1].replace(/\/\/.*$/gm, '').trim(); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + // Try to find raw JSON object with agent-like properties + const jsonMatch = output.match(/\{[\s\S]*"name"[\s\S]*\}/); + if (jsonMatch) { + try { + const jsonStr = jsonMatch[0].replace(/\/\/.*$/gm, ''); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + return null; +} + +export class AgentConfigCheck implements Check { + type = 'agent-config'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { agentResponse, projectDir } = context; + const expectedValid = this.config.expectedValid ?? true; + + let agentData: object | null = null; + + if (this.config.target === 'file' && this.config.filePath) { + try { + const filePath = path.join(projectDir, this.config.filePath); + const content = await fs.readFile(filePath, 'utf-8'); + const jsonStr = content.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, ''); + agentData = JSON.parse(jsonStr); + } catch (error) { + return { + name: this.config.description, + passed: !expectedValid, + details: `Failed to read agent file: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } else { + agentData = extractJsonFromOutput(agentResponse.output); + } + + if (!agentData) { + return { + name: this.config.description, + passed: !expectedValid, + details: 'No agent configuration found in output', + }; + } + + const result = AgentConfigSchema.safeParse(agentData); + + if (result.success) { + return { + name: this.config.description, + passed: expectedValid, + details: expectedValid ? 'Valid agent configuration' : 'Expected invalid but got valid configuration', + }; + } else { + const errors = result.error.errors.map(e => `${e.path.join('.')}: ${e.message}`).join(', '); + return { + name: this.config.description, + passed: !expectedValid, + details: expectedValid ? `Invalid agent configuration: ${errors}` : 'Correctly identified as invalid', + }; + } + } +} diff --git a/evals/src/checks/command-passes.ts b/evals/src/checks/command-passes.ts new file mode 100644 index 0000000..d536176 --- /dev/null +++ b/evals/src/checks/command-passes.ts @@ -0,0 +1,53 @@ +import { execa } from 'execa'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +export class CommandPassesCheck implements Check { + type = 'command-passes'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { projectDir } = context; + const command = this.config.command; + + if (!command) { + return { + name: this.config.description, + passed: false, + details: 'No command specified in check config', + }; + } + + try { + const [cmd, ...args] = command.split(' '); + const result = await execa(cmd, args, { + cwd: projectDir, + timeout: 120000, // 2 minute timeout + reject: false, + }); + + if (result.exitCode === 0) { + return { + name: this.config.description, + passed: true, + details: `Command "${command}" passed`, + }; + } else { + return { + name: this.config.description, + passed: false, + details: `Command "${command}" failed with exit code ${result.exitCode}: ${result.stderr || result.stdout}`.substring(0, 500), + }; + } + } catch (error) { + return { + name: this.config.description, + passed: false, + details: `Command failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } +} diff --git a/evals/src/checks/contains.ts b/evals/src/checks/contains.ts new file mode 100644 index 0000000..35d861f --- /dev/null +++ b/evals/src/checks/contains.ts @@ -0,0 +1,31 @@ +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +export class ContainsCheck implements Check { + type = 'contains'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { agentResponse } = context; + const searchValue = this.config.value ?? ''; + + if (!searchValue) { + return { + name: this.config.description, + passed: false, + details: 'No search value specified in check config', + }; + } + + const found = agentResponse.output.toLowerCase().includes(searchValue.toLowerCase()); + + return { + name: this.config.description, + passed: found, + details: found ? 'Found in output' : `"${searchValue}" not found in output`, + }; + } +} diff --git a/evals/src/checks/entity-config.ts b/evals/src/checks/entity-config.ts new file mode 100644 index 0000000..b66c112 --- /dev/null +++ b/evals/src/checks/entity-config.ts @@ -0,0 +1,128 @@ +import { z } from 'zod'; +import fs from 'fs/promises'; +import path from 'path'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +// Entity schema based on base44-cli entity structure +const PropertySchema = z.object({ + type: z.enum(['string', 'number', 'boolean', 'array', 'object']), + description: z.string().optional(), + items: z.object({ + type: z.string(), + }).optional(), + properties: z.record(z.any()).optional(), +}); + +const RLSConditionSchema = z.object({ + field: z.string(), + operator: z.enum(['eq', 'neq', 'in', 'nin', 'exists']), + value: z.union([z.string(), z.number(), z.boolean(), z.array(z.any())]), +}).or(z.object({ + any: z.array(z.any()), +})).or(z.object({ + all: z.array(z.any()), +})); + +const RLSSchema = z.object({ + read: RLSConditionSchema.optional(), + create: RLSConditionSchema.optional(), + update: RLSConditionSchema.optional(), + delete: RLSConditionSchema.optional(), +}); + +const EntitySchema = z.object({ + name: z.string().min(1), + type: z.literal('object').optional(), + description: z.string().optional(), + properties: z.record(PropertySchema), + required: z.array(z.string()).optional(), + rls: RLSSchema.optional(), +}); + +function extractJsonFromOutput(output: string): object | null { + // Try to find JSON in code blocks first + const codeBlockMatch = output.match(/```(?:json|jsonc)?\s*([\s\S]*?)```/); + if (codeBlockMatch) { + try { + // Remove comments for JSONC + const jsonStr = codeBlockMatch[1].replace(/\/\/.*$/gm, '').trim(); + return JSON.parse(jsonStr); + } catch { + // Continue to try other methods + } + } + + // Try to find raw JSON object + const jsonMatch = output.match(/\{[\s\S]*"name"[\s\S]*"properties"[\s\S]*\}/); + if (jsonMatch) { + try { + const jsonStr = jsonMatch[0].replace(/\/\/.*$/gm, ''); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + return null; +} + +export class EntityConfigCheck implements Check { + type = 'entity-config'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { agentResponse, projectDir } = context; + const expectedValid = this.config.expectedValid ?? true; + + let entityData: object | null = null; + + if (this.config.target === 'file' && this.config.filePath) { + // Read from file + try { + const filePath = path.join(projectDir, this.config.filePath); + const content = await fs.readFile(filePath, 'utf-8'); + // Remove JSONC comments + const jsonStr = content.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, ''); + entityData = JSON.parse(jsonStr); + } catch (error) { + return { + name: this.config.description, + passed: !expectedValid, + details: `Failed to read entity file: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } else { + // Extract from output + entityData = extractJsonFromOutput(agentResponse.output); + } + + if (!entityData) { + return { + name: this.config.description, + passed: !expectedValid, + details: 'No entity configuration found in output', + }; + } + + const result = EntitySchema.safeParse(entityData); + + if (result.success) { + return { + name: this.config.description, + passed: expectedValid, + details: expectedValid ? 'Valid entity schema' : 'Expected invalid but got valid schema', + }; + } else { + const errors = result.error.errors.map(e => `${e.path.join('.')}: ${e.message}`).join(', '); + return { + name: this.config.description, + passed: !expectedValid, + details: expectedValid ? `Invalid entity schema: ${errors}` : 'Correctly identified as invalid', + }; + } + } +} diff --git a/evals/src/checks/file-content.ts b/evals/src/checks/file-content.ts new file mode 100644 index 0000000..9f8d5bb --- /dev/null +++ b/evals/src/checks/file-content.ts @@ -0,0 +1,70 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +export class FileContentCheck implements Check { + type = 'file-content'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { projectDir } = context; + const filePath = this.config.filePath; + const pattern = this.config.pattern; + const value = this.config.value; + + if (!filePath) { + return { + name: this.config.description, + passed: false, + details: 'No file path specified in check config', + }; + } + + const fullPath = path.join(projectDir, filePath); + + try { + const content = await fs.readFile(fullPath, 'utf-8'); + + // Check for regex pattern + if (pattern) { + const regex = new RegExp(pattern, 'i'); + const matches = regex.test(content); + return { + name: this.config.description, + passed: matches, + details: matches + ? `Pattern "${pattern}" found in ${filePath}` + : `Pattern "${pattern}" not found in ${filePath}`, + }; + } + + // Check for substring + if (value) { + const contains = content.includes(value); + return { + name: this.config.description, + passed: contains, + details: contains + ? `"${value}" found in ${filePath}` + : `"${value}" not found in ${filePath}`, + }; + } + + return { + name: this.config.description, + passed: false, + details: 'No pattern or value specified for content check', + }; + } catch (error) { + return { + name: this.config.description, + passed: false, + details: `Failed to read file: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } +} diff --git a/evals/src/checks/file-exists.ts b/evals/src/checks/file-exists.ts new file mode 100644 index 0000000..549694d --- /dev/null +++ b/evals/src/checks/file-exists.ts @@ -0,0 +1,42 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +export class FileExistsCheck implements Check { + type = 'file-exists'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { projectDir } = context; + const filePath = this.config.filePath; + + if (!filePath) { + return { + name: this.config.description, + passed: false, + details: 'No file path specified in check config', + }; + } + + const fullPath = path.join(projectDir, filePath); + + try { + await fs.access(fullPath); + return { + name: this.config.description, + passed: true, + details: `File exists: ${filePath}`, + }; + } catch { + return { + name: this.config.description, + passed: false, + details: `File not found: ${filePath}`, + }; + } + } +} diff --git a/evals/src/checks/function-def.ts b/evals/src/checks/function-def.ts new file mode 100644 index 0000000..677205b --- /dev/null +++ b/evals/src/checks/function-def.ts @@ -0,0 +1,97 @@ +import { z } from 'zod'; +import fs from 'fs/promises'; +import path from 'path'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +// Function definition schema based on base44-cli function structure +const FunctionConfigSchema = z.object({ + name: z.string().min(1), + description: z.string().optional(), + entrypoint: z.string().optional(), + env: z.array(z.string()).optional(), + envFrom: z.array(z.string()).optional(), +}); + +function extractJsonFromOutput(output: string): object | null { + // Try to find JSON in code blocks first + const codeBlockMatch = output.match(/```(?:json|jsonc)?\s*([\s\S]*?)```/); + if (codeBlockMatch) { + try { + const jsonStr = codeBlockMatch[1].replace(/\/\/.*$/gm, '').trim(); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + // Try to find raw JSON object with function-like properties + const jsonMatch = output.match(/\{[\s\S]*"name"[\s\S]*\}/); + if (jsonMatch) { + try { + const jsonStr = jsonMatch[0].replace(/\/\/.*$/gm, ''); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + return null; +} + +export class FunctionDefCheck implements Check { + type = 'function-def'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { agentResponse, projectDir } = context; + const expectedValid = this.config.expectedValid ?? true; + + let functionData: object | null = null; + + if (this.config.target === 'file' && this.config.filePath) { + try { + const filePath = path.join(projectDir, this.config.filePath); + const content = await fs.readFile(filePath, 'utf-8'); + const jsonStr = content.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, ''); + functionData = JSON.parse(jsonStr); + } catch (error) { + return { + name: this.config.description, + passed: !expectedValid, + details: `Failed to read function file: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } else { + functionData = extractJsonFromOutput(agentResponse.output); + } + + if (!functionData) { + return { + name: this.config.description, + passed: !expectedValid, + details: 'No function configuration found in output', + }; + } + + const result = FunctionConfigSchema.safeParse(functionData); + + if (result.success) { + return { + name: this.config.description, + passed: expectedValid, + details: expectedValid ? 'Valid function definition' : 'Expected invalid but got valid definition', + }; + } else { + const errors = result.error.errors.map(e => `${e.path.join('.')}: ${e.message}`).join(', '); + return { + name: this.config.description, + passed: !expectedValid, + details: expectedValid ? `Invalid function definition: ${errors}` : 'Correctly identified as invalid', + }; + } + } +} diff --git a/evals/src/checks/interface.ts b/evals/src/checks/interface.ts new file mode 100644 index 0000000..5f2e6aa --- /dev/null +++ b/evals/src/checks/interface.ts @@ -0,0 +1,45 @@ +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +export type { Check, CheckConfig, CheckResult, EvalContext }; + +// Registry of check implementations +const checkRegistry: Map Check> = new Map(); + +export function registerCheck(type: string, factory: (config: CheckConfig) => Check): void { + checkRegistry.set(type, factory); +} + +export function createCheck(config: CheckConfig): Check { + const factory = checkRegistry.get(config.type); + if (!factory) { + throw new Error(`Unknown check type: ${config.type}`); + } + return factory(config); +} + +export function getRegisteredCheckTypes(): string[] { + return Array.from(checkRegistry.keys()); +} + +// Initialize checks - this will be called when the module is loaded +export async function initializeChecks(): Promise { + const { EntityConfigCheck } = await import('./entity-config.js'); + const { FunctionDefCheck } = await import('./function-def.js'); + const { AgentConfigCheck } = await import('./agent-config.js'); + const { ContainsCheck } = await import('./contains.js'); + const { JsonSchemaCheck } = await import('./json-schema.js'); + const { FileExistsCheck } = await import('./file-exists.js'); + const { FileContentCheck } = await import('./file-content.js'); + const { CommandPassesCheck } = await import('./command-passes.js'); + const { ValidJsonCheck } = await import('./valid-json.js'); + + registerCheck('entity-config', (config) => new EntityConfigCheck(config)); + registerCheck('function-def', (config) => new FunctionDefCheck(config)); + registerCheck('agent-config', (config) => new AgentConfigCheck(config)); + registerCheck('contains', (config) => new ContainsCheck(config)); + registerCheck('json-schema', (config) => new JsonSchemaCheck(config)); + registerCheck('file-exists', (config) => new FileExistsCheck(config)); + registerCheck('file-content', (config) => new FileContentCheck(config)); + registerCheck('command-passes', (config) => new CommandPassesCheck(config)); + registerCheck('valid-json', (config) => new ValidJsonCheck(config)); +} diff --git a/evals/src/checks/json-schema.ts b/evals/src/checks/json-schema.ts new file mode 100644 index 0000000..baa453c --- /dev/null +++ b/evals/src/checks/json-schema.ts @@ -0,0 +1,133 @@ +import { z } from 'zod'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +function extractJsonFromOutput(output: string): unknown | null { + // Try to find JSON in code blocks first + const codeBlockMatch = output.match(/```(?:json|jsonc)?\s*([\s\S]*?)```/); + if (codeBlockMatch) { + try { + const jsonStr = codeBlockMatch[1].replace(/\/\/.*$/gm, '').trim(); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + // Try to find raw JSON + const jsonMatch = output.match(/\{[\s\S]*\}/); + if (jsonMatch) { + try { + const jsonStr = jsonMatch[0].replace(/\/\/.*$/gm, ''); + return JSON.parse(jsonStr); + } catch { + // Continue + } + } + + // Try array + const arrayMatch = output.match(/\[[\s\S]*\]/); + if (arrayMatch) { + try { + return JSON.parse(arrayMatch[0]); + } catch { + // Continue + } + } + + return null; +} + +// Simple JSON schema to Zod converter for basic schemas +function jsonSchemaToZod(schema: Record): z.ZodType { + const type = schema.type as string; + + switch (type) { + case 'string': + return z.string(); + case 'number': + case 'integer': + return z.number(); + case 'boolean': + return z.boolean(); + case 'array': { + const items = schema.items as Record | undefined; + if (items) { + return z.array(jsonSchemaToZod(items)); + } + return z.array(z.unknown()); + } + case 'object': { + const properties = schema.properties as Record> | undefined; + const required = schema.required as string[] | undefined; + + if (properties) { + const shape: Record = {}; + for (const [key, propSchema] of Object.entries(properties)) { + const zodType = jsonSchemaToZod(propSchema); + shape[key] = required?.includes(key) ? zodType : zodType.optional(); + } + return z.object(shape); + } + return z.record(z.unknown()); + } + default: + return z.unknown(); + } +} + +export class JsonSchemaCheck implements Check { + type = 'json-schema'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { agentResponse } = context; + + if (!this.config.schema) { + return { + name: this.config.description, + passed: false, + details: 'No schema specified in check config', + }; + } + + const data = extractJsonFromOutput(agentResponse.output); + + if (data === null) { + return { + name: this.config.description, + passed: false, + details: 'No JSON found in output', + }; + } + + try { + const zodSchema = jsonSchemaToZod(this.config.schema); + const result = zodSchema.safeParse(data); + + if (result.success) { + return { + name: this.config.description, + passed: true, + details: 'Output matches schema', + }; + } else { + const errors = result.error.errors.map(e => `${e.path.join('.')}: ${e.message}`).join(', '); + return { + name: this.config.description, + passed: false, + details: `Schema validation failed: ${errors}`, + }; + } + } catch (error) { + return { + name: this.config.description, + passed: false, + details: `Schema conversion error: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } +} diff --git a/evals/src/checks/valid-json.ts b/evals/src/checks/valid-json.ts new file mode 100644 index 0000000..2383e97 --- /dev/null +++ b/evals/src/checks/valid-json.ts @@ -0,0 +1,49 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { Check, CheckConfig, CheckResult, EvalContext } from '../types.js'; + +export class ValidJsonCheck implements Check { + type = 'valid-json'; + private config: CheckConfig; + + constructor(config: CheckConfig) { + this.config = config; + } + + async run(context: EvalContext): Promise { + const { projectDir } = context; + const filePath = this.config.filePath; + + if (!filePath) { + return { + name: this.config.description, + passed: false, + details: 'No file path specified in check config', + }; + } + + const fullPath = path.join(projectDir, filePath); + + try { + const content = await fs.readFile(fullPath, 'utf-8'); + // Remove JSONC comments + const jsonStr = content + .replace(/\/\/.*$/gm, '') + .replace(/\/\*[\s\S]*?\*\//g, ''); + + JSON.parse(jsonStr); + + return { + name: this.config.description, + passed: true, + details: `Valid JSON in ${filePath}`, + }; + } catch (error) { + return { + name: this.config.description, + passed: false, + details: `Invalid JSON in ${filePath}: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } +} diff --git a/evals/src/compare-runner.ts b/evals/src/compare-runner.ts new file mode 100644 index 0000000..87394eb --- /dev/null +++ b/evals/src/compare-runner.ts @@ -0,0 +1,200 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; +import type { + CodingAgent, + ComparisonResult, + ExperimentRunResult, + FixtureResult, +} from './types.js'; +import { discoverExperiments, filterExperiments } from './experiments.js'; +import { setupFixtures } from './setup.js'; +import { runEvals, discoverFixtures } from './runner.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.join(__dirname, '..'); + +export interface CompareOptions { + name?: string; + experimentsDir?: string; + fixturesDir?: string; + verbose?: boolean; + filterFixtures?: string; + filterExperiments?: string; +} + +/** + * Run comparison across multiple experiments. + */ +export async function runComparison( + agent: CodingAgent, + options: CompareOptions = {} +): Promise { + const { + name = 'comparison', + experimentsDir = path.join(rootDir, 'experiments'), + fixturesDir = path.join(rootDir, 'fixtures'), + verbose = false, + filterFixtures, + filterExperiments: experimentFilter, + } = options; + + // Discover experiments + let experiments = await discoverExperiments(experimentsDir); + + if (experiments.length === 0) { + throw new Error(`No experiments found in ${experimentsDir}`); + } + + // Apply experiment filter + if (experimentFilter) { + experiments = filterExperiments(experiments, experimentFilter); + if (experiments.length === 0) { + throw new Error(`No experiments match filter: ${experimentFilter}`); + } + } + + if (verbose) { + console.log(`Found ${experiments.length} experiments: ${experiments.map(e => e.name).join(', ')}`); + } + + // Discover fixtures to get list of fixture names + let fixturesList = await discoverFixtures(fixturesDir); + if (filterFixtures) { + const filterLower = filterFixtures.toLowerCase(); + fixturesList = fixturesList.filter(f => f.toLowerCase().includes(filterLower)); + } + + // Extract fixture names from paths + const fixtureNames = fixturesList.map(f => { + const parts = f.split(path.sep); + return parts[parts.length - 1]; + }); + + if (verbose) { + console.log(`Found ${fixtureNames.length} fixtures`); + } + + // Initialize result structures + const experimentResults: ExperimentRunResult[] = []; + const matrix: Record> = {}; + + // Run each experiment + for (const experiment of experiments) { + if (verbose) { + console.log(`\n${'='.repeat(50)}`); + console.log(`Running experiment: ${experiment.name}`); + console.log(`Skills: ${experiment.skills.join(', ')}`); + console.log('='.repeat(50)); + } + + // Setup fixtures with this experiment's CLAUDE.md (if any) + await setupFixtures({ + fixturesDir, + experimentDir: experiment.skillsDir, + verbose, + }); + + // Run evals + const evalResult = await runEvals(agent, { + name: experiment.name, + fixturesDir, + verbose, + filter: filterFixtures, + }); + + // Convert to ExperimentRunResult + const expResult: ExperimentRunResult = { + experiment: experiment.name, + date: evalResult.date, + agent: evalResult.agent, + suites: evalResult.suites, + totalPassed: evalResult.totalPassed, + totalFailed: evalResult.totalFailed, + }; + experimentResults.push(expResult); + + // Build matrix entry for this experiment + matrix[experiment.name] = {}; + for (const suite of evalResult.suites) { + for (const fixture of suite.fixtures) { + matrix[experiment.name][fixture.name] = fixture.passed; + } + } + + if (verbose) { + console.log(`\nExperiment ${experiment.name}: ${evalResult.totalPassed} passed, ${evalResult.totalFailed} failed`); + } + } + + // Build summary + const summary = buildSummary(experimentResults, matrix); + + // Get all unique fixture names from results + const allFixtureNames = new Set(); + for (const exp of experimentResults) { + for (const suite of exp.suites) { + for (const fixture of suite.fixtures) { + allFixtureNames.add(fixture.name); + } + } + } + + return { + name, + date: new Date(), + agent: agent.name, + experiments: experiments.map(e => e.name), + fixtures: Array.from(allFixtureNames).sort(), + matrix, + experimentResults, + summary, + }; +} + +function buildSummary( + experimentResults: ExperimentRunResult[], + matrix: Record> +): ComparisonResult['summary'] { + const byExperiment: Record = {}; + const byFixture: Record = {}; + + // Calculate per-experiment stats + for (const result of experimentResults) { + byExperiment[result.experiment] = { + passed: result.totalPassed, + failed: result.totalFailed, + total: result.totalPassed + result.totalFailed, + }; + } + + // Calculate per-fixture stats + for (const [expName, fixtures] of Object.entries(matrix)) { + for (const [fixtureName, passed] of Object.entries(fixtures)) { + if (!byFixture[fixtureName]) { + byFixture[fixtureName] = { passedExperiments: [], failedExperiments: [] }; + } + if (passed) { + byFixture[fixtureName].passedExperiments.push(expName); + } else { + byFixture[fixtureName].failedExperiments.push(expName); + } + } + } + + // Find best experiment + let bestExperiment = ''; + let bestPassRate = -1; + for (const [expName, stats] of Object.entries(byExperiment)) { + const passRate = stats.total > 0 ? stats.passed / stats.total : 0; + if (passRate > bestPassRate) { + bestPassRate = passRate; + bestExperiment = expName; + } + } + + return { + byExperiment, + byFixture, + bestExperiment, + }; +} diff --git a/evals/src/compare.ts b/evals/src/compare.ts new file mode 100644 index 0000000..483f721 --- /dev/null +++ b/evals/src/compare.ts @@ -0,0 +1,264 @@ +#!/usr/bin/env node + +import path from 'path'; +import { createAgent } from './agents/interface.js'; +import { runComparison } from './compare-runner.js'; +import { writeComparisonReport } from './reporters/comparison-markdown.js'; +import type { ComparisonResult } from './types.js'; + +interface CliOptions { + name: string; + agent: string; + experimentsDir: string; + fixturesDir: string; + outputDir: string; + verbose: boolean; + runs: number; + filterFixtures?: string; + filterExperiments?: string; +} + +function parseArgs(args: string[]): CliOptions { + const options: CliOptions = { + name: 'comparison', + agent: 'claude-code', + experimentsDir: 'experiments', + fixturesDir: 'fixtures', + outputDir: 'results', + verbose: false, + runs: 1, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + switch (arg) { + case '--name': + case '-n': + options.name = args[++i] ?? 'comparison'; + break; + case '--agent': + case '-a': + options.agent = args[++i] ?? 'claude-code'; + break; + case '--experiments': + case '-e': + options.experimentsDir = args[++i] ?? 'experiments'; + break; + case '--fixtures': + case '-f': + options.fixturesDir = args[++i] ?? 'fixtures'; + break; + case '--output': + case '-o': + options.outputDir = args[++i] ?? 'results'; + break; + case '--verbose': + case '-v': + options.verbose = true; + break; + case '--runs': + case '-r': + options.runs = parseInt(args[++i] ?? '1', 10); + break; + case '--filter-fixtures': + options.filterFixtures = args[++i]; + break; + case '--filter-experiments': + options.filterExperiments = args[++i]; + break; + case '--help': + case '-h': + printHelp(); + process.exit(0); + } + } + + return options; +} + +function printHelp(): void { + console.log(` +Experiment Comparison Runner + +Usage: npm run eval:compare -- [options] + +Options: + -n, --name Name for this comparison run (default: "comparison") + -r, --runs Number of times to run (default: 1) + -a, --agent Agent to use (default: "claude-code") + -e, --experiments Experiments directory (default: "experiments") + -f, --fixtures Fixtures directory (default: "fixtures") + -o, --output Output directory for reports (default: "results") + -v, --verbose Verbose output + --filter-fixtures Filter fixtures by name pattern + --filter-experiments Filter experiments by name pattern + -h, --help Show this help + +Examples: + npm run eval:compare + npm run eval:compare -- --runs 3 # Run 3 times to check consistency + npm run eval:compare -- --name "skills-test" --verbose + npm run eval:compare -- --filter-experiments baseline + npm run eval:compare -- --filter-fixtures entities +`); +} + +interface RunResult { + runNumber: number; + result: ComparisonResult; + reportPath: string; +} + +interface AggregatedStats { + experiment: string; + runs: number; + totalPassed: number; + totalFailed: number; + perfectRuns: number; // Runs with 100% pass rate + passRates: number[]; // Pass rate per run + avgPassRate: number; + consistency: string; // "3/3" format +} + +function aggregateResults(runResults: RunResult[]): Map { + const stats = new Map(); + + for (const { runNumber, result } of runResults) { + for (const [expName, expStats] of Object.entries(result.summary.byExperiment)) { + if (!stats.has(expName)) { + stats.set(expName, { + experiment: expName, + runs: 0, + totalPassed: 0, + totalFailed: 0, + perfectRuns: 0, + passRates: [], + avgPassRate: 0, + consistency: '', + }); + } + + const s = stats.get(expName)!; + s.runs++; + s.totalPassed += expStats.passed; + s.totalFailed += expStats.failed; + + const passRate = expStats.total > 0 ? (expStats.passed / expStats.total) * 100 : 0; + s.passRates.push(passRate); + + if (expStats.failed === 0) { + s.perfectRuns++; + } + } + } + + // Calculate averages and consistency + for (const s of stats.values()) { + s.avgPassRate = s.passRates.reduce((a, b) => a + b, 0) / s.passRates.length; + s.consistency = `${s.perfectRuns}/${s.runs}`; + } + + return stats; +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + + console.log(`Starting comparison: ${options.name}`); + console.log(`Agent: ${options.agent}`); + console.log(`Runs: ${options.runs}`); + console.log(`Experiments: ${options.experimentsDir}`); + console.log(`Fixtures: ${options.fixturesDir}`); + console.log(''); + + const agent = createAgent(options.agent); + const runResults: RunResult[] = []; + + for (let run = 1; run <= options.runs; run++) { + if (options.runs > 1) { + console.log(''); + console.log('#'.repeat(50)); + console.log(`# Run ${run}/${options.runs}`); + console.log('#'.repeat(50)); + } + + const result = await runComparison(agent, { + name: `${options.name}-run${run}`, + experimentsDir: options.experimentsDir, + fixturesDir: options.fixturesDir, + verbose: options.verbose, + filterFixtures: options.filterFixtures, + filterExperiments: options.filterExperiments, + }); + + const reportPath = await writeComparisonReport(result, options.outputDir); + runResults.push({ runNumber: run, result, reportPath }); + + // Print run summary + console.log(''); + console.log(`Run ${run} Results:`); + for (const [expName, stats] of Object.entries(result.summary.byExperiment)) { + const passRate = stats.total > 0 ? Math.round((stats.passed / stats.total) * 100) : 0; + const status = stats.failed === 0 ? '✅' : '❌'; + console.log(` ${status} ${expName}: ${stats.passed}/${stats.total} (${passRate}%)`); + } + } + + // Print aggregated summary for multiple runs + if (options.runs > 1) { + const aggregated = aggregateResults(runResults); + + console.log(''); + console.log('='.repeat(60)); + console.log(`AGGREGATED RESULTS (${options.runs} runs)`); + console.log('='.repeat(60)); + console.log(''); + console.log('| Experiment | Avg Pass Rate | Consistency | Perfect Runs |'); + console.log('|------------|---------------|-------------|--------------|'); + + // Sort by consistency (perfect runs) then by avg pass rate + const sorted = Array.from(aggregated.values()).sort((a, b) => { + if (b.perfectRuns !== a.perfectRuns) return b.perfectRuns - a.perfectRuns; + return b.avgPassRate - a.avgPassRate; + }); + + for (const s of sorted) { + const avgStr = `${Math.round(s.avgPassRate)}%`; + const marker = s.perfectRuns === s.runs ? ' ⭐' : ''; + console.log(`| ${s.experiment.padEnd(10)} | ${avgStr.padEnd(13)} | ${s.consistency.padEnd(11)} | ${s.perfectRuns}/${s.runs}${marker.padEnd(10)} |`); + } + + console.log(''); + console.log('Legend: ⭐ = 100% consistency (all runs passed)'); + console.log(''); + console.log('Reports:'); + for (const { runNumber, reportPath } of runResults) { + console.log(` Run ${runNumber}: ${path.relative(process.cwd(), reportPath)}`); + } + } else { + // Single run summary + const result = runResults[0].result; + console.log(''); + console.log('='.repeat(50)); + console.log('Comparison Complete'); + console.log('='.repeat(50)); + console.log(`Experiments: ${result.experiments.length}`); + console.log(`Fixtures: ${result.fixtures.length}`); + console.log(`Best: ${result.summary.bestExperiment}`); + console.log(''); + console.log(`Report: ${path.relative(process.cwd(), runResults[0].reportPath)}`); + } + + // Exit with error code if any run had failures in best experiment + const lastResult = runResults[runResults.length - 1].result; + const bestStats = lastResult.summary.byExperiment[lastResult.summary.bestExperiment]; + if (bestStats && bestStats.failed > 0) { + process.exit(1); + } +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); diff --git a/evals/src/experiments.ts b/evals/src/experiments.ts new file mode 100644 index 0000000..8a60376 --- /dev/null +++ b/evals/src/experiments.ts @@ -0,0 +1,60 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { Experiment } from './types.js'; + +/** + * Discover experiments from the experiments directory. + * Each subdirectory is an experiment. Experiments can optionally contain CLAUDE.md. + * Skills are provided by the fixtures themselves, not by experiments. + */ +export async function discoverExperiments(experimentsDir: string): Promise { + const experiments: Experiment[] = []; + + try { + const entries = await fs.readdir(experimentsDir, { withFileTypes: true }); + + for (const entry of entries) { + if (!entry.isDirectory() || entry.name.startsWith('.')) { + continue; + } + + const experimentDir = path.join(experimentsDir, entry.name); + const hasClaude = await hasClaudeMd(experimentDir); + + experiments.push({ + name: entry.name, + skillsDir: experimentDir, + skills: hasClaude ? ['CLAUDE.md'] : [], + }); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return []; + } + throw error; + } + + return experiments.sort((a, b) => a.name.localeCompare(b.name)); +} + +/** + * Check if experiment directory has a CLAUDE.md file. + */ +async function hasClaudeMd(experimentDir: string): Promise { + try { + await fs.access(path.join(experimentDir, 'CLAUDE.md')); + return true; + } catch { + return false; + } +} + +/** + * Filter experiments by name pattern. + */ +export function filterExperiments(experiments: Experiment[], pattern: string): Experiment[] { + const patternLower = pattern.toLowerCase(); + return experiments.filter(exp => + exp.name.toLowerCase().includes(patternLower) + ); +} diff --git a/evals/src/index.ts b/evals/src/index.ts new file mode 100644 index 0000000..cf06b4a --- /dev/null +++ b/evals/src/index.ts @@ -0,0 +1,145 @@ +#!/usr/bin/env node + +import path from 'path'; +import { createAgent } from './agents/interface.js'; +import { runEvals } from './runner.js'; +import { writeReport } from './reporters/markdown.js'; + +interface CliOptions { + name: string; + agent: string; + fixturesDir: string; + outputDir: string; + verbose: boolean; + filter?: string; + fixtures?: string[]; +} + +function parseArgs(args: string[]): CliOptions { + const options: CliOptions = { + name: 'default', + agent: 'claude-code', + fixturesDir: 'fixtures', + outputDir: 'results', + verbose: false, + }; + + const positionalArgs: string[] = []; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (!arg.startsWith('-')) { + positionalArgs.push(arg); + continue; + } + + switch (arg) { + case '--name': + case '-n': + options.name = args[++i] ?? 'default'; + break; + case '--agent': + case '-a': + options.agent = args[++i] ?? 'claude-code'; + break; + case '--fixtures': + case '-f': + options.fixturesDir = args[++i] ?? 'fixtures'; + break; + case '--output': + case '-o': + options.outputDir = args[++i] ?? 'results'; + break; + case '--verbose': + case '-v': + options.verbose = true; + break; + case '--filter': + options.filter = args[++i]; + break; + case '--help': + case '-h': + printHelp(); + process.exit(0); + } + } + + if (positionalArgs.length > 0) { + options.fixtures = positionalArgs; + } + + return options; +} + +function printHelp(): void { + console.log(` +Skills Eval Runner + +Usage: npm run eval -- [options] [fixture-names...] + +Arguments: + fixture-names Specific fixtures to run (by name) + +Options: + -n, --name Name for this eval run (default: "default") + -a, --agent Agent to use (default: "claude-code") + -f, --fixtures Fixtures directory (default: "fixtures") + -o, --output Output directory for reports (default: "results") + -v, --verbose Verbose output + --filter Filter fixtures by name pattern + -h, --help Show this help + +Examples: + npm run eval + npm run eval -- --name "baseline" + npm run eval -- --name "sonnet-test" --verbose + npm run eval -- --filter entities + npm run eval -- skill-check nextjs-todo + npm run eval -- agent-chat-component --verbose +`); +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + + console.log(`Starting eval run: ${options.name}`); + console.log(`Agent: ${options.agent}`); + console.log(`Fixtures: ${options.fixturesDir}`); + if (options.fixtures) { + console.log(`Running specific fixtures: ${options.fixtures.join(', ')}`); + } + console.log(''); + + const agent = createAgent(options.agent); + + const result = await runEvals(agent, { + name: options.name, + fixturesDir: options.fixturesDir, + verbose: options.verbose, + filter: options.filter, + fixtures: options.fixtures, + }); + + // Write report + const reportPath = await writeReport(result, options.outputDir); + + // Print summary + console.log(''); + console.log('='.repeat(50)); + console.log('Eval Run Complete'); + console.log('='.repeat(50)); + console.log(`Passed: ${result.totalPassed}`); + console.log(`Failed: ${result.totalFailed}`); + console.log(`Report: ${path.relative(process.cwd(), reportPath)}`); + + // Exit with error code if any tests failed + if (result.totalFailed > 0) { + process.exit(1); + } +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); diff --git a/evals/src/reporters/comparison-markdown.ts b/evals/src/reporters/comparison-markdown.ts new file mode 100644 index 0000000..6400f02 --- /dev/null +++ b/evals/src/reporters/comparison-markdown.ts @@ -0,0 +1,202 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { ComparisonResult, ExperimentRunResult, FixtureResult } from '../types.js'; + +function formatDate(date: Date): string { + return date.toISOString().replace('T', ' ').substring(0, 19); +} + +function formatDateForFilename(date: Date): string { + return date.toISOString() + .replace('T', '-') + .replace(/:/g, '') + .substring(0, 17); +} + +function escapeMarkdown(text: string): string { + return text.replace(/[|]/g, '\\|'); +} + +function generateSummaryTable(result: ComparisonResult): string { + const lines: string[] = []; + + lines.push('## Summary'); + lines.push('| Experiment | Passed | Failed | Pass Rate |'); + lines.push('|------------|--------|--------|-----------|'); + + // Sort by pass rate descending + const sortedExperiments = Object.entries(result.summary.byExperiment) + .sort((a, b) => { + const rateA = a[1].total > 0 ? a[1].passed / a[1].total : 0; + const rateB = b[1].total > 0 ? b[1].passed / b[1].total : 0; + return rateB - rateA; + }); + + for (const [expName, stats] of sortedExperiments) { + const passRate = stats.total > 0 + ? Math.round((stats.passed / stats.total) * 100) + : 0; + const isBest = expName === result.summary.bestExperiment; + const name = isBest ? `**${expName}**` : expName; + lines.push(`| ${name} | ${stats.passed} | ${stats.failed} | ${passRate}% |`); + } + + lines.push(''); + return lines.join('\n'); +} + +function generateComparisonMatrix(result: ComparisonResult): string { + const lines: string[] = []; + + lines.push('## Comparison Matrix'); + + // Header row + const headerCells = ['Fixture', ...result.experiments]; + lines.push(`| ${headerCells.join(' | ')} |`); + lines.push(`|${headerCells.map(() => '---').join('|')}|`); + + // Fixture rows + for (const fixture of result.fixtures) { + const cells = [fixture]; + for (const exp of result.experiments) { + const passed = result.matrix[exp]?.[fixture]; + cells.push(passed === undefined ? '-' : passed ? '✅' : '❌'); + } + lines.push(`| ${cells.join(' | ')} |`); + } + + lines.push(''); + return lines.join('\n'); +} + +function generateInconsistentResults(result: ComparisonResult): string { + const lines: string[] = []; + + // Find fixtures with inconsistent results (some passed, some failed) + const inconsistent: Array<{ + fixture: string; + passed: string[]; + failed: string[]; + }> = []; + + for (const [fixtureName, stats] of Object.entries(result.summary.byFixture)) { + if (stats.passedExperiments.length > 0 && stats.failedExperiments.length > 0) { + inconsistent.push({ + fixture: fixtureName, + passed: stats.passedExperiments, + failed: stats.failedExperiments, + }); + } + } + + if (inconsistent.length === 0) { + return ''; + } + + lines.push('## Inconsistent Results'); + lines.push(''); + + for (const item of inconsistent) { + lines.push(`- **${item.fixture}**: Passed in ${item.passed.join(', ')}, Failed in ${item.failed.join(', ')}`); + } + + lines.push(''); + return lines.join('\n'); +} + +function generateExperimentDetails(expResult: ExperimentRunResult): string { + const lines: string[] = []; + const total = expResult.totalPassed + expResult.totalFailed; + + lines.push(`
`); + lines.push(`${expResult.experiment} (${expResult.totalPassed}/${total})`); + lines.push(''); + + for (const suite of expResult.suites) { + lines.push(`### ${suite.name}`); + lines.push(''); + + for (const fixture of suite.fixtures) { + const status = fixture.passed ? '✅' : '❌'; + lines.push(`- ${status} **${fixture.name}**: ${escapeMarkdown(fixture.description)}`); + + // Show failed checks + if (!fixture.passed) { + const failedChecks = fixture.checks.filter(c => !c.passed); + for (const check of failedChecks) { + lines.push(` - ❌ ${escapeMarkdown(check.name)}: ${escapeMarkdown(check.details ?? '')}`); + } + if (fixture.error) { + lines.push(` - Error: ${escapeMarkdown(fixture.error)}`); + } + if (!fixture.skillCheckPassed) { + lines.push(` - Skills: expected ${fixture.expectedSkills.join(', ')}, got ${fixture.skillsInvoked.join(', ') || '(none)'}`); + } + } + } + + lines.push(''); + } + + lines.push('
'); + lines.push(''); + + return lines.join('\n'); +} + +export function generateComparisonReport(result: ComparisonResult): string { + const lines: string[] = []; + + // Header + lines.push(`# Experiment Comparison: ${result.name}`); + lines.push(`**Date**: ${formatDate(result.date)}`); + lines.push(`**Agent**: ${result.agent}`); + lines.push(`**Experiments**: ${result.experiments.length} | **Fixtures**: ${result.fixtures.length}`); + lines.push(''); + + // Summary table + lines.push(generateSummaryTable(result)); + + // Comparison matrix + lines.push(generateComparisonMatrix(result)); + + // Inconsistent results + const inconsistent = generateInconsistentResults(result); + if (inconsistent) { + lines.push(inconsistent); + } + + // Detailed results per experiment + lines.push('## Detailed Results'); + lines.push(''); + + for (const expResult of result.experimentResults) { + lines.push(generateExperimentDetails(expResult)); + } + + return lines.join('\n'); +} + +export async function writeComparisonReport( + result: ComparisonResult, + outputDir: string +): Promise { + await fs.mkdir(outputDir, { recursive: true }); + + const report = generateComparisonReport(result); + const filename = `comparison-${formatDateForFilename(result.date)}-${result.name}.md`; + const filepath = path.join(outputDir, filename); + + await fs.writeFile(filepath, report, 'utf-8'); + + // Update latest comparison symlink + const latestPath = path.join(outputDir, 'latest-comparison.md'); + try { + await fs.unlink(latestPath); + } catch { + // Ignore if doesn't exist + } + await fs.symlink(filename, latestPath); + + return filepath; +} diff --git a/evals/src/reporters/markdown.ts b/evals/src/reporters/markdown.ts new file mode 100644 index 0000000..192dd98 --- /dev/null +++ b/evals/src/reporters/markdown.ts @@ -0,0 +1,139 @@ +import fs from 'fs/promises'; +import path from 'path'; +import type { EvalRunResult, FixtureResult, SuiteResult } from '../types.js'; + +function formatDate(date: Date): string { + return date.toISOString().replace('T', ' ').substring(0, 19); +} + +function formatDateForFilename(date: Date): string { + return date.toISOString() + .replace('T', '-') + .replace(/:/g, '') + .substring(0, 17); +} + +function escapeMarkdown(text: string): string { + return text.replace(/[|]/g, '\\|'); +} + +function generateFixtureSection(fixture: FixtureResult): string { + const status = fixture.passed ? '✅' : '❌'; + const lines: string[] = []; + + lines.push(`### ${status} ${fixture.name}`); + lines.push(`**Fixture**: \`${fixture.fixtureDir}\``); + lines.push(`**Prompt**: ${escapeMarkdown(fixture.prompt)}`); + lines.push(''); + + // Skills section + const skillStatus = fixture.skillCheckPassed ? '✓' : '✗'; + if (fixture.expectedSkills.length > 0) { + lines.push(`- **Expected Skills**: ${fixture.expectedSkills.join(', ')} ${skillStatus}`); + } + if (fixture.skillsInvoked.length > 0) { + lines.push(`- **Skills Invoked**: ${fixture.skillsInvoked.join(', ')}`); + } else { + lines.push('- **Skills Invoked**: (none detected)'); + } + lines.push(''); + + // Checks table + if (fixture.checks.length > 0) { + lines.push('- **Checks**:'); + lines.push(' | Check | Status | Details |'); + lines.push(' |-------|--------|---------|'); + for (const check of fixture.checks) { + const checkStatus = check.passed ? '✅' : '❌'; + const details = escapeMarkdown(check.details ?? ''); + lines.push(` | ${escapeMarkdown(check.name)} | ${checkStatus} | ${details} |`); + } + lines.push(''); + } + + // Error if present + if (fixture.error) { + lines.push(`> **Error**: ${escapeMarkdown(fixture.error)}`); + lines.push(''); + } + + // Agent output in details + lines.push('
'); + lines.push('Agent Output'); + lines.push(''); + lines.push('```'); + lines.push(fixture.agentOutput.substring(0, 5000)); // Truncate long outputs + if (fixture.agentOutput.length > 5000) { + lines.push('... (truncated)'); + } + lines.push('```'); + lines.push('
'); + lines.push(''); + + return lines.join('\n'); +} + +function generateSuiteSection(suite: SuiteResult): string { + const lines: string[] = []; + + lines.push(`## ${suite.name} Suite`); + lines.push(''); + + for (const fixture of suite.fixtures) { + lines.push(generateFixtureSection(fixture)); + } + + return lines.join('\n'); +} + +export function generateMarkdownReport(result: EvalRunResult): string { + const lines: string[] = []; + + // Header + lines.push(`# Eval Run: ${result.name}`); + lines.push(`**Date**: ${formatDate(result.date)}`); + lines.push(`**Agent**: ${result.agent}`); + + const totalFixtures = result.totalPassed + result.totalFailed; + lines.push(`**Fixtures**: ${totalFixtures}`); + lines.push(''); + + // Summary table + lines.push('## Summary'); + lines.push('| Status | Count |'); + lines.push('|--------|-------|'); + lines.push(`| ✅ Passed | ${result.totalPassed} |`); + lines.push(`| ❌ Failed | ${result.totalFailed} |`); + lines.push(''); + + // Suite sections + for (const suite of result.suites) { + lines.push(generateSuiteSection(suite)); + } + + return lines.join('\n'); +} + +export async function writeReport( + result: EvalRunResult, + outputDir: string +): Promise { + await fs.mkdir(outputDir, { recursive: true }); + + const report = generateMarkdownReport(result); + const filename = `run-${formatDateForFilename(result.date)}-${result.name}.md`; + const filepath = path.join(outputDir, filename); + + await fs.writeFile(filepath, report, 'utf-8'); + + // Update latest symlink + const latestPath = path.join(outputDir, 'latest.md'); + try { + await fs.unlink(latestPath); + } catch { + // Ignore if doesn't exist + } + await fs.symlink(filename, latestPath); + + return filepath; +} diff --git a/evals/src/runner.ts b/evals/src/runner.ts new file mode 100644 index 0000000..b571907 --- /dev/null +++ b/evals/src/runner.ts @@ -0,0 +1,278 @@ +import fs from 'fs/promises'; +import path from 'path'; +import os from 'os'; +import type { + CodingAgent, + EvalConfig, + EvalRunResult, + FixtureResult, + SuiteResult, +} from './types.js'; +import { createCheck, initializeChecks } from './checks/interface.js'; + +async function copyDir(src: string, dest: string): Promise { + await fs.mkdir(dest, { recursive: true }); + const entries = await fs.readdir(src, { withFileTypes: true }); + + for (const entry of entries) { + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + + if (entry.isDirectory()) { + await copyDir(srcPath, destPath); + } else if (entry.isSymbolicLink()) { + const link = await fs.readlink(srcPath); + await fs.symlink(link, destPath); + } else { + await fs.copyFile(srcPath, destPath); + } + } +} + +async function loadEvalConfig(fixtureDir: string): Promise { + const configPath = path.join(fixtureDir, 'eval.json'); + const content = await fs.readFile(configPath, 'utf-8'); + return JSON.parse(content) as EvalConfig; +} + +async function runFixture( + agent: CodingAgent, + fixtureDir: string, + verbose: boolean +): Promise { + const config = await loadEvalConfig(fixtureDir); + + // Create temp directory and copy fixture + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'eval-')); + await copyDir(fixtureDir, tempDir); + + // Determine project directory (default to fixture root if no project/ subdirectory) + let projectDir = tempDir; + try { + const projectSubdir = path.join(tempDir, 'project'); + const stat = await fs.stat(projectSubdir); + if (stat.isDirectory()) { + projectDir = projectSubdir; + } + } catch { + // No project/ subdirectory, use fixture root + } + + if (verbose) { + console.log(` Running fixture: ${config.name}`); + console.log(` Temp dir: ${tempDir}`); + console.log(` Project dir: ${projectDir}`); + } + + try { + // Run the agent + const agentResponse = await agent.run(config.prompt, projectDir); + + if (verbose) { + console.log(` Skills invoked: ${agentResponse.skillsInvoked.join(', ') || '(none)'}`); + console.log(` --- Agent Output ---`); + console.log(agentResponse.output); + console.log(` --- End Output ---`); + } + + // Check skill activation + const skillCheckPassed = config.expectedSkills.every( + skill => agentResponse.skillsInvoked.includes(skill) + ); + + // Run all checks + const checkResults = await Promise.all( + config.checks.map(async checkConfig => { + const check = createCheck(checkConfig); + return check.run({ + agentResponse, + fixtureDir, + projectDir, + checkConfig, + }); + }) + ); + + // Determine overall pass/fail + const allChecksPassed = checkResults.every(r => r.passed); + const passed = skillCheckPassed && allChecksPassed; + + return { + name: config.name, + description: config.description, + fixtureDir: path.relative(process.cwd(), fixtureDir), + prompt: config.prompt, + expectedSkills: config.expectedSkills, + skillsInvoked: agentResponse.skillsInvoked, + skillCheckPassed, + checks: checkResults, + passed, + agentOutput: agentResponse.output, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { + name: config.name, + description: config.description, + fixtureDir: path.relative(process.cwd(), fixtureDir), + prompt: config.prompt, + expectedSkills: config.expectedSkills, + skillsInvoked: [], + skillCheckPassed: false, + checks: [], + passed: false, + agentOutput: '', + error: errorMessage, + }; + } finally { + // Clean up temp directory + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // Ignore cleanup errors + } + } +} + +export async function discoverFixtures(baseDir: string): Promise { + const fixtures: string[] = []; + + async function scan(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + // Check if this directory is a fixture (has eval.json) + const hasEvalJson = entries.some(e => e.isFile() && e.name === 'eval.json'); + if (hasEvalJson) { + fixtures.push(dir); + return; // Don't recurse into fixtures + } + + // Recurse into subdirectories + for (const entry of entries) { + if (entry.isDirectory() && !entry.name.startsWith('.')) { + await scan(path.join(dir, entry.name)); + } + } + } + + await scan(baseDir); + return fixtures.sort(); +} + +function groupFixturesBySuite(fixtures: string[]): Map { + const suites = new Map(); + + for (const fixture of fixtures) { + // Extract suite name from path (e.g., fixtures/entities/basic-entity -> entities) + const parts = fixture.split(path.sep); + const fixturesIndex = parts.indexOf('fixtures'); + const suiteName = fixturesIndex >= 0 && parts.length > fixturesIndex + 1 + ? parts[fixturesIndex + 1] + : 'default'; + + if (!suites.has(suiteName)) { + suites.set(suiteName, []); + } + suites.get(suiteName)!.push(fixture); + } + + return suites; +} + +export interface RunOptions { + name?: string; + fixturesDir?: string; + verbose?: boolean; + filter?: string; + fixtures?: string[]; +} + +export async function runEvals( + agent: CodingAgent, + options: RunOptions = {} +): Promise { + const { + name = 'default', + fixturesDir = 'fixtures', + verbose = false, + filter, + fixtures: fixtureNames, + } = options; + + // Initialize check registry + await initializeChecks(); + + // Discover fixtures + let fixtures = await discoverFixtures(fixturesDir); + + // Filter by specific fixture names if provided + if (fixtureNames && fixtureNames.length > 0) { + fixtures = fixtures.filter(f => { + const fixtureName = path.basename(f); + return fixtureNames.some(name => + fixtureName === name || fixtureName.includes(name) + ); + }); + } + + // Apply filter pattern if specified + if (filter) { + const filterLower = filter.toLowerCase(); + fixtures = fixtures.filter(f => f.toLowerCase().includes(filterLower)); + } + + if (verbose) { + console.log(`Found ${fixtures.length} fixtures`); + } + + // Group by suite + const suiteMap = groupFixturesBySuite(fixtures); + const suites: SuiteResult[] = []; + + let totalPassed = 0; + let totalFailed = 0; + + for (const [suiteName, suiteFixtures] of suiteMap) { + if (verbose) { + console.log(`\nRunning suite: ${suiteName}`); + } + + const fixtureResults: FixtureResult[] = []; + + for (const fixtureDir of suiteFixtures) { + const result = await runFixture(agent, fixtureDir, verbose); + fixtureResults.push(result); + + if (result.passed) { + totalPassed++; + if (verbose) { + console.log(` ✅ ${result.name}`); + } + } else { + totalFailed++; + if (verbose) { + console.log(` ❌ ${result.name}`); + if (result.error) { + console.log(` Error: ${result.error}`); + } + } + } + } + + suites.push({ + name: suiteName, + fixtures: fixtureResults, + passed: fixtureResults.filter(f => f.passed).length, + failed: fixtureResults.filter(f => !f.passed).length, + }); + } + + return { + name, + date: new Date(), + agent: agent.name, + suites, + totalPassed, + totalFailed, + }; +} diff --git a/evals/src/setup.ts b/evals/src/setup.ts new file mode 100644 index 0000000..9c92f85 --- /dev/null +++ b/evals/src/setup.ts @@ -0,0 +1,156 @@ +#!/usr/bin/env node + +/** + * Setup script that copies skills from fixture's own skills folder to project/. + * Experiments can optionally provide CLAUDE.md to override behavior. + */ + +import fs from 'fs/promises'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.join(__dirname, '..'); +const defaultFixturesDir = path.join(rootDir, 'fixtures'); + +async function copyDir(src: string, dest: string): Promise { + await fs.mkdir(dest, { recursive: true }); + const entries = await fs.readdir(src, { withFileTypes: true }); + + for (const entry of entries) { + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + + if (entry.isDirectory()) { + await copyDir(srcPath, destPath); + } else if (entry.isSymbolicLink()) { + const link = await fs.readlink(srcPath); + try { + await fs.unlink(destPath); + } catch { + // Ignore if doesn't exist + } + await fs.symlink(link, destPath); + } else { + await fs.copyFile(srcPath, destPath); + } + } +} + +export interface SetupOptions { + fixturesDir?: string; + experimentDir?: string; // Optional experiment dir for CLAUDE.md override + verbose?: boolean; +} + +export async function setupFixtures(options: SetupOptions = {}): Promise { + const { + fixturesDir = defaultFixturesDir, + experimentDir, + verbose = true, + } = options; + + if (verbose) { + console.log('Setting up fixtures...'); + } + + // Get list of fixtures + const fixtureDirs = await fs.readdir(fixturesDir, { withFileTypes: true }); + const fixtures = fixtureDirs + .filter(d => d.isDirectory() && !d.name.startsWith('.')) + .map(d => d.name); + + if (verbose) { + console.log(`Found fixtures: ${fixtures.join(', ')}`); + } + + // Copy skills from each fixture's own skills folder to project/skills + for (const fixture of fixtures) { + const fixtureDir = path.join(fixturesDir, fixture); + const fixtureSkillsSrc = path.join(fixtureDir, 'skills'); + const projectSkillsDest = path.join(fixtureDir, 'project', 'skills'); + + // Check if fixture has its own skills folder + let skillCount = 0; + try { + await fs.access(fixtureSkillsSrc); + + // Get list of skills in fixture + const skillDirs = await fs.readdir(fixtureSkillsSrc, { withFileTypes: true }); + const skills = skillDirs + .filter(d => d.isDirectory() && !d.name.startsWith('.')) + .map(d => d.name); + skillCount = skills.length; + + // Remove existing project skills directory + try { + await fs.rm(projectSkillsDest, { recursive: true, force: true }); + } catch { + // Ignore + } + + // Copy fixture's skills to project/skills + await copyDir(fixtureSkillsSrc, projectSkillsDest); + } catch { + // No skills folder in fixture, skip + if (verbose) { + console.log(` ⚠ ${fixture}: no skills folder found`); + } + continue; + } + + // Handle CLAUDE.md from experiment (if provided) + const destClaudeMd = path.join(fixtureDir, 'project', 'CLAUDE.md'); + let hasClaude = false; + + if (experimentDir) { + const srcClaudeMd = path.join(experimentDir, 'CLAUDE.md'); + try { + await fs.access(srcClaudeMd); + await fs.copyFile(srcClaudeMd, destClaudeMd); + hasClaude = true; + } catch { + // No CLAUDE.md in experiment, remove any existing one + try { + await fs.unlink(destClaudeMd); + } catch { + // Ignore if doesn't exist + } + } + } else { + // No experiment, remove any existing CLAUDE.md + try { + await fs.unlink(destClaudeMd); + } catch { + // Ignore if doesn't exist + } + } + + if (verbose) { + const suffix = hasClaude ? ' + CLAUDE.md' : ''; + console.log(` ✓ ${fixture}: copied ${skillCount} skills${suffix}`); + } + } + + if (verbose) { + console.log('Setup complete!'); + } +} + +async function setup(): Promise { + try { + await setupFixtures(); + } catch (error) { + console.error('Setup failed:', error instanceof Error ? error.message : error); + process.exit(1); + } +} + +// Run if executed directly +const isMainModule = process.argv[1]?.includes('setup'); +if (isMainModule) { + setup().catch(error => { + console.error('Setup failed:', error); + process.exit(1); + }); +} diff --git a/evals/src/types.ts b/evals/src/types.ts new file mode 100644 index 0000000..e9f2794 --- /dev/null +++ b/evals/src/types.ts @@ -0,0 +1,111 @@ +export interface CodingAgentResponse { + output: string; + skillsInvoked: string[]; + metadata?: Record; +} + +export interface CodingAgent { + name: string; + run(prompt: string, workingDir: string): Promise; +} + +export interface CheckResult { + name: string; + passed: boolean; + details?: string; +} + +export interface EvalContext { + agentResponse: CodingAgentResponse; + fixtureDir: string; + projectDir: string; + checkConfig: CheckConfig; +} + +export interface Check { + type: string; + run(context: EvalContext): Promise; +} + +export interface CheckConfig { + type: string; + description: string; + target?: 'output' | 'file'; + filePath?: string; + expectedValid?: boolean; + value?: string; + pattern?: string; + command?: string; + schema?: Record; +} + +export interface EvalConfig { + name: string; + description: string; + prompt: string; + expectedSkills: string[]; + checks: CheckConfig[]; +} + +export interface FixtureResult { + name: string; + description: string; + fixtureDir: string; + prompt: string; + expectedSkills: string[]; + skillsInvoked: string[]; + skillCheckPassed: boolean; + checks: CheckResult[]; + passed: boolean; + agentOutput: string; + error?: string; +} + +export interface SuiteResult { + name: string; + fixtures: FixtureResult[]; + passed: number; + failed: number; +} + +export interface EvalRunResult { + name: string; + date: Date; + agent: string; + suites: SuiteResult[]; + totalPassed: number; + totalFailed: number; +} + +// Experiment configuration +export interface Experiment { + name: string; // Directory name + skillsDir: string; // Absolute path to skills + skills: string[]; // Skill names found +} + +// Result for one experiment across all fixtures +export interface ExperimentRunResult { + experiment: string; + date: Date; + agent: string; + suites: SuiteResult[]; + totalPassed: number; + totalFailed: number; +} + +// Full comparison result +export interface ComparisonResult { + name: string; + date: Date; + agent: string; + experiments: string[]; + fixtures: string[]; + matrix: Record>; // exp -> fixture -> passed + experimentResults: ExperimentRunResult[]; + summary: { + byExperiment: Record; + byFixture: Record; + bestExperiment: string; + }; +} diff --git a/evals/tsconfig.json b/evals/tsconfig.json new file mode 100644 index 0000000..d99b7ee --- /dev/null +++ b/evals/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} From d38727448231b23d0373f1f4f8d40c81341f9c36 Mon Sep 17 00:00:00 2001 From: Gal Elmalah Date: Wed, 4 Feb 2026 10:43:24 +0200 Subject: [PATCH 3/3] push skills level experiments --- evals/experiments/explicit-read/CLAUDE.md | 8 ++++ evals/experiments/strict-read/CLAUDE.md | 18 +++++++++ evals/src/experiments.ts | 34 +++++++++++++---- evals/src/setup.ts | 46 ++++++++++++++++++----- 4 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 evals/experiments/explicit-read/CLAUDE.md create mode 100644 evals/experiments/strict-read/CLAUDE.md diff --git a/evals/experiments/explicit-read/CLAUDE.md b/evals/experiments/explicit-read/CLAUDE.md new file mode 100644 index 0000000..bbdc7da --- /dev/null +++ b/evals/experiments/explicit-read/CLAUDE.md @@ -0,0 +1,8 @@ +# Base44 Development Guidelines + +When working with Base44 projects, always read the relevant skill documentation before making changes: + +- Before using Base44 CLI commands, read `skills/base44-cli/SKILL.md` +- Before working with Base44 SDK (entities, agents, etc.), read `skills/base44-sdk/SKILL.md` + +This ensures you follow the correct patterns and avoid common mistakes. diff --git a/evals/experiments/strict-read/CLAUDE.md b/evals/experiments/strict-read/CLAUDE.md new file mode 100644 index 0000000..38f1987 --- /dev/null +++ b/evals/experiments/strict-read/CLAUDE.md @@ -0,0 +1,18 @@ +# Base44 Development Guidelines + +## IMPORTANT: Read Skills Before Any Base44 Work + +Before performing ANY task involving Base44, you MUST: + +1. **Read ALL skill documentation first:** + - `skills/base44-cli/SKILL.md` - CLI commands and project management + - `skills/base44-sdk/SKILL.md` - SDK usage, entities, and agents + +2. **Do not skip this step.** The skills contain critical information about: + - Correct syntax and patterns + - Common mistakes to avoid + - Required file formats and structures + +3. **Apply the patterns from the skills exactly.** Do not improvise or use different approaches than what the skills document. + +Failure to read the skills first will result in incorrect implementations. diff --git a/evals/src/experiments.ts b/evals/src/experiments.ts index 8a60376..7e5945a 100644 --- a/evals/src/experiments.ts +++ b/evals/src/experiments.ts @@ -4,8 +4,9 @@ import type { Experiment } from './types.js'; /** * Discover experiments from the experiments directory. - * Each subdirectory is an experiment. Experiments can optionally contain CLAUDE.md. - * Skills are provided by the fixtures themselves, not by experiments. + * Each subdirectory is an experiment. Experiments can contain: + * - CLAUDE.md: Override the project's CLAUDE.md + * - skills/: Directory of skills to use instead of fixture skills */ export async function discoverExperiments(experimentsDir: string): Promise { const experiments: Experiment[] = []; @@ -19,12 +20,12 @@ export async function discoverExperiments(experimentsDir: string): Promise { +async function getExperimentContents(experimentDir: string): Promise { + const contents: string[] = []; + + // Check for CLAUDE.md try { await fs.access(path.join(experimentDir, 'CLAUDE.md')); - return true; + contents.push('CLAUDE.md'); + } catch { + // No CLAUDE.md + } + + // Check for skills directory + try { + const skillsDir = path.join(experimentDir, 'skills'); + const skillEntries = await fs.readdir(skillsDir, { withFileTypes: true }); + const skillNames = skillEntries + .filter(d => d.isDirectory() && !d.name.startsWith('.')) + .map(d => d.name); + contents.push(...skillNames); } catch { - return false; + // No skills directory } + + return contents; } /** diff --git a/evals/src/setup.ts b/evals/src/setup.ts index 9c92f85..72edb2c 100644 --- a/evals/src/setup.ts +++ b/evals/src/setup.ts @@ -39,7 +39,7 @@ async function copyDir(src: string, dest: string): Promise { export interface SetupOptions { fixturesDir?: string; - experimentDir?: string; // Optional experiment dir for CLAUDE.md override + experimentDir?: string; // Optional experiment dir for CLAUDE.md and skills override verbose?: boolean; } @@ -64,19 +64,44 @@ export async function setupFixtures(options: SetupOptions = {}): Promise { console.log(`Found fixtures: ${fixtures.join(', ')}`); } - // Copy skills from each fixture's own skills folder to project/skills + // Determine skills source: experiment skills override fixture skills + let experimentSkillsSrc: string | null = null; + let experimentSkillNames: string[] = []; + if (experimentDir) { + const expSkillsPath = path.join(experimentDir, 'skills'); + try { + await fs.access(expSkillsPath); + const skillDirs = await fs.readdir(expSkillsPath, { withFileTypes: true }); + experimentSkillNames = skillDirs + .filter(d => d.isDirectory() && !d.name.startsWith('.')) + .map(d => d.name); + if (experimentSkillNames.length > 0) { + experimentSkillsSrc = expSkillsPath; + if (verbose) { + console.log(`Using experiment skills: ${experimentSkillNames.join(', ')}`); + } + } + } catch { + // No skills in experiment, will use fixture skills + } + } + + // Copy skills to each fixture's project/skills for (const fixture of fixtures) { const fixtureDir = path.join(fixturesDir, fixture); const fixtureSkillsSrc = path.join(fixtureDir, 'skills'); const projectSkillsDest = path.join(fixtureDir, 'project', 'skills'); - // Check if fixture has its own skills folder + // Use experiment skills if available, otherwise fixture skills + const skillsSrc = experimentSkillsSrc ?? fixtureSkillsSrc; + + // Check if skills source exists let skillCount = 0; try { - await fs.access(fixtureSkillsSrc); + await fs.access(skillsSrc); - // Get list of skills in fixture - const skillDirs = await fs.readdir(fixtureSkillsSrc, { withFileTypes: true }); + // Get list of skills + const skillDirs = await fs.readdir(skillsSrc, { withFileTypes: true }); const skills = skillDirs .filter(d => d.isDirectory() && !d.name.startsWith('.')) .map(d => d.name); @@ -89,10 +114,10 @@ export async function setupFixtures(options: SetupOptions = {}): Promise { // Ignore } - // Copy fixture's skills to project/skills - await copyDir(fixtureSkillsSrc, projectSkillsDest); + // Copy skills to project/skills + await copyDir(skillsSrc, projectSkillsDest); } catch { - // No skills folder in fixture, skip + // No skills folder found, skip if (verbose) { console.log(` ⚠ ${fixture}: no skills folder found`); } @@ -128,7 +153,8 @@ export async function setupFixtures(options: SetupOptions = {}): Promise { if (verbose) { const suffix = hasClaude ? ' + CLAUDE.md' : ''; - console.log(` ✓ ${fixture}: copied ${skillCount} skills${suffix}`); + const source = experimentSkillsSrc ? ' (from experiment)' : ''; + console.log(` ✓ ${fixture}: copied ${skillCount} skills${source}${suffix}`); } }