diff --git a/apps/registry-docs/app/api/registry/contents/route.ts b/apps/registry-docs/app/api/registry/contents/route.ts
index 8a1a711d..c54e7cb7 100644
--- a/apps/registry-docs/app/api/registry/contents/route.ts
+++ b/apps/registry-docs/app/api/registry/contents/route.ts
@@ -31,7 +31,7 @@ export async function GET() {
const providerBaseUrl =
provider.meta?.registryUrl ??
c.root.meta?.registryUrl ??
- `https://github.com/514-labs/factory/tree/main/connector-registry/${c.connectorId}/${version}/${provider.authorId}`;
+ `https://github.com/514-labs/registry/tree/main/connector-registry/${c.connectorId}/${version}/${provider.authorId}`;
const githubUrl =
impl.implementation === "default"
@@ -84,7 +84,7 @@ export async function GET() {
const providerBaseUrl =
p.root.meta?.registryUrl ??
- `https://github.com/514-labs/factory/tree/main/pipeline-registry/${p.pipelineId}/${version}/${provider.authorId}`;
+ `https://github.com/514-labs/registry/tree/main/pipeline-registry/${p.pipelineId}/${version}/${provider.authorId}`;
const githubUrl =
impl.implementation === "default"
diff --git a/apps/registry-docs/app/connectors/[connector]/[version]/[creator]/[language]/[implementation]/page.tsx b/apps/registry-docs/app/connectors/[connector]/[version]/[creator]/[language]/[implementation]/page.tsx
index d6f09ef7..37b96783 100644
--- a/apps/registry-docs/app/connectors/[connector]/[version]/[creator]/[language]/[implementation]/page.tsx
+++ b/apps/registry-docs/app/connectors/[connector]/[version]/[creator]/[language]/[implementation]/page.tsx
@@ -98,7 +98,7 @@ export default async function ConnectorImplementationPage({
const registryUrl =
provider.meta?.registryUrl ??
meta?.registryUrl ??
- `https://github.com/514-labs/factory/tree/main/connector-registry/${connector}/${version}/${creator}`;
+ `https://github.com/514-labs/registry/tree/main/connector-registry/${connector}/${version}/${creator}`;
// Get issue URL for current language/implementation
const issueValue = provider.meta?.issues?.[implEntry.language];
@@ -107,7 +107,7 @@ export default async function ConnectorImplementationPage({
? issueValue
: issueValue && typeof issueValue === "object"
? (issueValue[implEntry.implementation] ?? issueValue["default"])
- : `https://github.com/514-labs/factory/issues`;
+ : `https://github.com/514-labs/registry/issues`;
// Build lists and navigation helpers
const getProviderVersion = (pPath: string): string =>
diff --git a/apps/registry-docs/app/docs/connectors/creating/page.tsx b/apps/registry-docs/app/docs/connectors/creating/page.tsx
index 0fcce0be..9c159074 100644
--- a/apps/registry-docs/app/docs/connectors/creating/page.tsx
+++ b/apps/registry-docs/app/docs/connectors/creating/page.tsx
@@ -13,8 +13,8 @@ export default function CreatingConnectorsPage() {
1. Clone the repository
Clone the factory repository to your local machine:
- {`git clone https://github.com/514-labs/factory.git
-cd factory`}
+ {`git clone https://github.com/514-labs/registry.git
+cd registry`}
2. Generate a connector scaffold
@@ -180,18 +180,18 @@ pnpm validate:schemas`}
8. Share your connector
Open a pull request to the{" "}
-
- factory repository
+
+ registry repository
{" "}
to share your connector with the community. Mention the connector{" "}
-
+
Issue
{" "}
in the PR description.
- If you built your connector outside the factory monorepo, you'll need to
+ If you built your connector outside the registry monorepo, you'll need to
add it to the monorepo's registry directory with all required metadata
(defined in `connector-registry/_scaffold`).
diff --git a/apps/registry-docs/app/docs/connectors/quickstart/page.tsx b/apps/registry-docs/app/docs/connectors/quickstart/page.tsx
index a662637d..dc898504 100644
--- a/apps/registry-docs/app/docs/connectors/quickstart/page.tsx
+++ b/apps/registry-docs/app/docs/connectors/quickstart/page.tsx
@@ -16,8 +16,8 @@ export default function ConnectorQuickstartPage() {
1. Clone the Repository
- {`git clone https://github.com/514-labs/factory.git
-cd factory`}
+ {`git clone https://github.com/514-labs/registry.git
+cd registry`}
2. Generate Your Connector
diff --git a/apps/registry-docs/app/docs/pipelines/creating/page.tsx b/apps/registry-docs/app/docs/pipelines/creating/page.tsx
index c4d30c68..4e0626ac 100644
--- a/apps/registry-docs/app/docs/pipelines/creating/page.tsx
+++ b/apps/registry-docs/app/docs/pipelines/creating/page.tsx
@@ -29,8 +29,8 @@ export default function CreatingPipelinesPage() {
1. Clone the Repository
- {`git clone https://github.com/514-labs/factory.git
-cd factory`}
+ {`git clone https://github.com/514-labs/registry.git
+cd registry`}
2. Generate Pipeline Scaffold
diff --git a/apps/registry-docs/app/pipelines/[pipeline]/[version]/[creator]/[language]/[implementation]/page.tsx b/apps/registry-docs/app/pipelines/[pipeline]/[version]/[creator]/[language]/[implementation]/page.tsx
index cbf04cf6..cdc21e13 100644
--- a/apps/registry-docs/app/pipelines/[pipeline]/[version]/[creator]/[language]/[implementation]/page.tsx
+++ b/apps/registry-docs/app/pipelines/[pipeline]/[version]/[creator]/[language]/[implementation]/page.tsx
@@ -88,7 +88,7 @@ export default async function PipelineImplementationPage({
// URLs
const registryUrl =
meta?.registryUrl ??
- `https://github.com/514-labs/factory/tree/main/pipeline-registry/${pipeline}/${version}/${creator}`;
+ `https://github.com/514-labs/registry/tree/main/pipeline-registry/${pipeline}/${version}/${creator}`;
// Build lists and navigation helpers
const getProviderVersion = (pPath: string): string =>
diff --git a/apps/registry-docs/app/registry.json/route.ts b/apps/registry-docs/app/registry.json/route.ts
index 7d2c3a40..655bf0ca 100644
--- a/apps/registry-docs/app/registry.json/route.ts
+++ b/apps/registry-docs/app/registry.json/route.ts
@@ -28,7 +28,7 @@ export async function GET() {
const providerBaseUrl =
provider.meta?.registryUrl ??
c.root.meta?.registryUrl ??
- `https://github.com/514-labs/factory/tree/main/connector-registry/${c.connectorId}/${version}/${provider.authorId}`;
+ `https://github.com/514-labs/registry/tree/main/connector-registry/${c.connectorId}/${version}/${provider.authorId}`;
const githubUrl =
impl.implementation === "default"
@@ -73,7 +73,7 @@ export async function GET() {
const providerBaseUrl =
p.root.meta?.registryUrl ??
- `https://github.com/514-labs/factory/tree/main/pipeline-registry/${p.pipelineId}/${version}/${provider.authorId}`;
+ `https://github.com/514-labs/registry/tree/main/pipeline-registry/${p.pipelineId}/${version}/${provider.authorId}`;
const githubUrl =
impl.implementation === "default"
diff --git a/apps/registry-docs/components/site-header.tsx b/apps/registry-docs/components/site-header.tsx
index e7cd619f..e577b062 100644
--- a/apps/registry-docs/components/site-header.tsx
+++ b/apps/registry-docs/components/site-header.tsx
@@ -46,7 +46,7 @@ export async function SiteHeader() {
('/contacts', send)
+ return {
+ listContacts: base.list,
+ getContact: base.get,
+ streamContacts: base.streamAll,
+ getContacts: base.getAll
+ }
+}
+```
-Every API connector must implement the following core functionality and structure:
-#### Resource Abstraction
+### Core Modules and Methods
-- Organize code by API resources rather than ETL stages.
-- Each resource module lives under `src/{resource}` and must expose:
- - `createResource(send)` factory that binds a base path (e.g., `/{resource}`) and returns a CRUD surface
- - Operations: `list(params)`, `get({ id, ... })`, `streamAll(params)`, `getAll(params)`
- - `model` definition describing the item shape for the resource
-- Cross-cutting helpers should live under `src/lib`:
- - `paginate` iterator supporting cursor pagination (and extensible for other strategies)
- - `make-resource` (or equivalent) to build the CRUD surface with pagination
+Every API connector must implement the following core functionality:
#### Initialization and Lifecycle
-- **initialize(configuration)**
- Sets up the connector with the provided configuration. Should validate the configuration and prepare any internal state.
+The `ApiConnectorBase` class provides these methods which your connector inherits:
+
+- **initialize(configuration, defaults, authFunction, rateLimitOptions)**
+ Extends the base initialization with your configuration defaults, authentication strategy, and rate limiting options.
- **connect()**
- Establishes connection to the API service. May include authentication, session creation, or connection pooling.
+ Inherited from base class - establishes connection state.
- **disconnect()**
- Gracefully closes the connection and cleans up resources. Should complete any pending requests before disconnecting.
+ Inherited from base class - gracefully closes connection.
- **isConnected()**
- Returns true if the connector is currently connected and ready to make requests, false otherwise.
+ Inherited from base class - returns connection status.
-#### Request Methods
+#### Domain Delegation Pattern
-- **request(options)**
- Core method for making HTTP requests. All other HTTP methods should internally use this method.
- Options should include: method, path, headers, query parameters, body, timeout, and any method-specific settings.
+Connectors expose methods that delegate to domain builders:
+
+```typescript
+private get domain() {
+ const sendLite: SendFn = async (args) => this.send(args)
+ return {
+ ...buildContactsDomain((args) => sendLite({ ...args, operation: 'contacts' })),
+ ...buildCompaniesDomain((args) => sendLite({ ...args, operation: 'companies' }))
+ }
+}
+
+// Public methods delegate to domains
+listContacts = (params?) => this.domain.listContacts(params)
+getContact = (params) => this.domain.getContact(params)
+```
+
+#### Request Methods
-- **get(path, options)**
- Performs an HTTP GET request to the specified path.
+The `ApiConnectorBase` provides the core request infrastructure:
-- **post(path, data, options)**
- Performs an HTTP POST request with the provided data payload.
+- **send(options)**
+ Protected method for making HTTP requests with full rate limiting, retry logic, and hooks.
+ Automatically handles authentication, rate limiting, and response wrapping.
-- **put(path, data, options)**
- Performs an HTTP PUT request to update a resource.
+- **request(options)**
+ Public alias for `send()` method - inherited from base class.
-- **patch(path, data, options)**
- Performs an HTTP PATCH request for partial updates.
+#### HTTP Client Integration
-- **delete(path, options)**
- Performs an HTTP DELETE request to remove a resource.
+The base class uses `HttpClient` from core package which provides:
+- Automatic retry with exponential backoff
+- Rate limiting with `TokenBucketLimiter`
+- Hook system for middleware
+- Response envelope wrapping
+- Error handling and classification
#### Advanced Operations
@@ -173,9 +242,19 @@ Hooks provide extension points for customizing connector behavior without modify
#### Hook Structure
+Hooks follow a standardized interface:
+
+```typescript
+interface Hook {
+ name: string
+ priority?: number
+ execute: (ctx: HookContext) => Promise | void
+}
+```
+
- **name** - Unique identifier for the hook
-- **priority** - Execution order (lower numbers execute first)
-- **execute(context)** - The hook's main function
+- **priority** - Execution order (lower numbers execute first)
+- **execute(context)** - The hook's main function with full context access
#### Hook Context
@@ -187,11 +266,26 @@ Each hook receives a context object containing:
- **error** - The error object (when applicable)
- **metadata** - Additional context data
-#### Context Methods
+#### Enhanced Hook Context
+
+```typescript
+interface HookContext {
+ type: HookType
+ operation?: string
+ request?: Record
+ response?: HttpResponseEnvelope
+ error?: unknown
+ metadata?: Record
+ modifyRequest?: (updates: Record) => void
+ modifyResponse?: (updates: Partial>) => void
+ abort?: (reason?: string) => void
+}
+```
- **modifyRequest(updates)** - Modify the outgoing request
-- **modifyResponse(updates)** - Modify the incoming response
+- **modifyResponse(updates)** - Modify the incoming response
- **abort(reason)** - Cancel the request with a reason
+- **operation** - Track which domain operation is executing
#### Middleware Pipeline (conceptual)
@@ -505,9 +599,35 @@ result = GET /jobs/{jobId}/result
### Observability
-- **Logging**: Structured logs with correlation `requestId`, redaction of secrets, and consistent fields.
-- **Metrics**: Counters (requests, errors, retries), distributions (latency, payload sizes), gauges (in‑flight, rate limits).
-- **Tracing**: Span per request with attributes for method, path, status, retryCount, rateLimit.
+The core package provides built-in observability hooks that connectors should export:
+
+```typescript
+// Export observability hooks for external use
+export { createLoggingHooks, createMetricsHooks, InMemoryMetricsSink, createInMemoryMetricsSink } from './observability'
+```
+
+#### Built-in Observability Features
+
+- **Logging Hooks**: Structured logs with correlation `requestId`, operation tracking, and secret redaction
+- **Metrics Hooks**: Counters (requests, errors, retries), histograms (latency), gauges (in‑flight requests)
+- **Request Correlation**: Every request gets a correlation ID for distributed tracing
+- **Rate Limit Tracking**: Real-time rate limit status in response metadata
+
+#### Observability Integration
+
+```typescript
+const connector = createMyConnector()
+const loggingHooks = createLoggingHooks()
+const metricsHooks = createMetricsHooks(metricsSink)
+
+connector.initialize({
+ auth: { /* auth config */ },
+ hooks: {
+ beforeRequest: [...loggingHooks.beforeRequest, ...metricsHooks.beforeRequest],
+ afterResponse: [...loggingHooks.afterResponse, ...metricsHooks.afterResponse]
+ }
+})
+```
### Security and Compliance
@@ -537,14 +657,31 @@ Connectors must include:
### Conformance Checklist
-- Implements lifecycle: initialize, connect, disconnect, isConnected
-- Provides request primitives, optional stream/upload/download when applicable
-- Config supports baseUrl, timeouts, proxy/tls, auth, retry, rate limit, defaults, hooks
-- Retry with backoff + jitter, honors Retry‑After, has circuit breaker and retry budget
-- Hook pipeline before/after/error/retry; deterministic order and cancellation
-- Response wrapper with data/status/headers/meta including requestId and rateLimit
-- Structured errors with code/status/retryable/details and correlation id
-- Pagination supports cursor/offset/page/link‑header with pluggable extractors
-- Concurrency limits, cancellation, graceful shutdown
-- Observability: logs/metrics/traces with redaction
-- Security controls for credentials, TLS, validation, and redaction
+#### Core Package Integration
+- ✅ Extends `ApiConnectorBase` from `@connector-factory/core`
+- ✅ Uses domain factory pattern with `buildResourceDomain(send)` functions
+- ✅ Implements proper initialization with defaults, auth, and rate limiting
+- ✅ Exports observability hooks (`createLoggingHooks`, `createMetricsHooks`)
+- ✅ Uses `makeCrudDomain` for consistent CRUD operations
+
+#### Architecture Compliance
+- ✅ Domain organization under `src/domains/{resource}.ts`
+- ✅ Model organization under `src/models/{resource}/`
+- ✅ Type exports for external usage (`export type *`)
+- ✅ Configuration with proper TypeScript types
+- ✅ Hook system with named hooks and priority
+
+#### Base Functionality (Inherited)
+- ✅ Lifecycle: initialize, connect, disconnect, isConnected
+- ✅ Request infrastructure with retry, rate limiting, hooks
+- ✅ Response wrapper with data/status/headers/meta including requestId and rateLimit
+- ✅ Structured errors with code/status/retryable/details and correlation
+- ✅ Pagination with cursor support via `paginateCursor`
+- ✅ Concurrency limits, cancellation, graceful shutdown
+- ✅ Security controls for credentials, TLS, validation, and redaction
+
+#### Testing Requirements
+- ✅ Unit tests using Jest framework
+- ✅ Integration tests with real API (gated)
+- ✅ Separate test configurations for unit vs integration
+- ✅ Hook testing and domain testing
diff --git a/apps/registry-docs/public/openapi.yaml b/apps/registry-docs/public/openapi.yaml
index bf7ce75a..0d8b7818 100644
--- a/apps/registry-docs/public/openapi.yaml
+++ b/apps/registry-docs/public/openapi.yaml
@@ -7,7 +7,7 @@ info:
version: 1.0.0
contact:
name: 514 Labs
- url: https://github.com/514-labs/factory
+ url: https://github.com/514-labs/registry
license:
name: MIT
url: https://opensource.org/licenses/MIT
diff --git a/connector-registry/_scaffold/python.json b/connector-registry/_scaffold/python.json
index b9fce9e3..0644434c 100644
--- a/connector-registry/_scaffold/python.json
+++ b/connector-registry/_scaffold/python.json
@@ -65,7 +65,7 @@
{
"type": "file",
"name": "connector.json",
- "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/connector.schema.json\",\n \"identifier\": \"{connector}\",\n \"name\": \"{connector}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"avatarUrlOverride\": \"\",\n \"version\": \"{version}\",\n \"language\": \"python\",\n \"implementation\": \"{implementation}\",\n \"tags\": [],\n \"category\": \"api\",\n \"description\": \"\",\n \"homepage\": \"\",\n \"license\": \"MIT\",\n \"source\": {\"type\":\"api\",\"spec\":\"\"},\n \"capabilities\": {\"extract\": true, \"transform\": true, \"load\": true},\n \"maintainers\": [],\n \"issues\": \"\",\n \"registryUrl\": \"https://github.com/514-labs/factory/tree/main/connector-registry/{connector}/{version}/{author}/python/{implementation}\"\n}\n"
+ "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/connector.schema.json\",\n \"identifier\": \"{connector}\",\n \"name\": \"{connector}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"avatarUrlOverride\": \"\",\n \"version\": \"{version}\",\n \"language\": \"python\",\n \"implementation\": \"{implementation}\",\n \"tags\": [],\n \"category\": \"api\",\n \"description\": \"\",\n \"homepage\": \"\",\n \"license\": \"MIT\",\n \"source\": {\"type\":\"api\",\"spec\":\"\"},\n \"capabilities\": {\"extract\": true, \"transform\": true, \"load\": true},\n \"maintainers\": [],\n \"issues\": \"\",\n \"registryUrl\": \"https://github.com/514-labs/registry/tree/main/connector-registry/{connector}/{version}/{author}/python/{implementation}\"\n}\n"
},
{
"type": "file",
diff --git a/connector-registry/_scaffold/typescript.json b/connector-registry/_scaffold/typescript.json
index 5e08b9bc..adfaa8fc 100644
--- a/connector-registry/_scaffold/typescript.json
+++ b/connector-registry/_scaffold/typescript.json
@@ -65,7 +65,7 @@
{
"type": "file",
"name": "connector.json",
- "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/connector.schema.json\",\n \"identifier\": \"{connector}\",\n \"name\": \"{connector}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"avatarUrlOverride\": \"\",\n \"version\": \"{version}\",\n \"language\": \"typescript\",\n \"implementation\": \"{implementation}\",\n \"tags\": [],\n \"category\": \"api\",\n \"description\": \"\",\n \"homepage\": \"\",\n \"license\": \"MIT\",\n \"source\": {\"type\":\"api\",\"spec\":\"\"},\n \"capabilities\": {\"extract\": true, \"transform\": true, \"load\": true},\n \"maintainers\": [],\n \"issues\": \"\",\n \"registryUrl\": \"https://github.com/514-labs/factory/tree/main/connector-registry/{connector}/{version}/{author}/typescript/{implementation}\"\n}\n"
+ "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/connector.schema.json\",\n \"identifier\": \"{connector}\",\n \"name\": \"{connector}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"avatarUrlOverride\": \"\",\n \"version\": \"{version}\",\n \"language\": \"typescript\",\n \"implementation\": \"{implementation}\",\n \"tags\": [],\n \"category\": \"api\",\n \"description\": \"\",\n \"homepage\": \"\",\n \"license\": \"MIT\",\n \"source\": {\"type\":\"api\",\"spec\":\"\"},\n \"capabilities\": {\"extract\": true, \"transform\": true, \"load\": true},\n \"maintainers\": [],\n \"issues\": \"\",\n \"registryUrl\": \"https://github.com/514-labs/registry/tree/main/connector-registry/{connector}/{version}/{author}/typescript/{implementation}\"\n}\n"
},
{
"type": "file",
@@ -99,17 +99,22 @@
{
"type": "file",
"name": "package.json",
- "template": "{\n \"name\": \"{packageName}\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"type\": \"module\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts --dts --format esm,cjs\",\n \"test\": \"vitest run\"\n },\n \"engines\": {\"node\": \">=20\"},\n \"dependencies\": {},\n \"devDependencies\": {\n \"tsup\": \"^8.0.0\",\n \"typescript\": \"^5.4.0\",\n \"vitest\": \"^1.6.0\"\n }\n}\n"
+ "template": "{\n \"name\": \"{packageName}\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"description\": \"TypeScript {connector} connector\",\n \"license\": \"MIT\",\n \"main\": \"dist/src/index.js\",\n \"types\": \"dist/src/index.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./dist/src/index.d.ts\",\n \"default\": \"./dist/src/index.js\"\n }\n },\n \"files\": [\"dist\", \"README.md\"],\n \"scripts\": {\n \"build\": \"tsc -p tsconfig.json\",\n \"test\": \"jest --passWithNoTests\",\n \"test:watch\": \"jest --watch\",\n \"test:integration\": \"jest --config jest.integration.cjs\"\n },\n \"dependencies\": {\n \"@connector-factory/core\": \"workspace:*\",\n \"ajv\": \"^8.17.1\",\n \"ajv-formats\": \"^3.0.1\",\n \"zod\": \"^3.25.8\"\n },\n \"devDependencies\": {\n \"@types/jest\": \"^29.5.14\",\n \"@types/node\": \"^20.14.12\",\n \"jest\": \"^29.7.0\",\n \"nock\": \"^13.5.4\",\n \"ts-jest\": \"29.1.2\",\n \"tsx\": \"^4.19.2\",\n \"typescript\": \"^5.6.3\"\n },\n \"engines\": {\"node\": \">=20\"}\n}\n"
},
{
"type": "file",
"name": "tsconfig.json",
- "template": "{\n \"compilerOptions\": {\n \"target\": \"ES2022\",\n \"module\": \"ESNext\",\n \"moduleResolution\": \"Bundler\",\n \"outDir\": \"dist\",\n \"declaration\": true,\n \"declarationMap\": true,\n \"strict\": true,\n \"skipLibCheck\": true\n },\n \"include\": [\"src\"]\n}\n"
+ "template": "{\n \"compilerOptions\": {\n \"target\": \"ES2022\",\n \"module\": \"CommonJS\",\n \"moduleResolution\": \"Node\",\n \"outDir\": \"dist\",\n \"declaration\": true,\n \"declarationMap\": true,\n \"strict\": true,\n \"skipLibCheck\": true,\n \"esModuleInterop\": true,\n \"allowSyntheticDefaultImports\": true,\n \"forceConsistentCasingInFileNames\": true\n },\n \"include\": [\"src\", \"tests\"],\n \"exclude\": [\"node_modules\", \"dist\"]\n}\n"
},
{
"type": "file",
- "name": "vitest.config.ts",
- "template": "import { defineConfig } from 'vitest/config'\nexport default defineConfig({ test: { environment: 'node' } })\n"
+ "name": "jest.config.cjs",
+ "template": "module.exports = {\n preset: 'ts-jest',\n testEnvironment: 'node',\n roots: ['/src', '/tests'],\n testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'],\n collectCoverageFrom: ['src/**/*.ts', '!src/**/*.d.ts'],\n setupFilesAfterEnv: [],\n moduleNameMapping: {},\n testTimeout: 30000\n};\n"
+ },
+ {
+ "type": "file",
+ "name": "jest.integration.cjs",
+ "template": "module.exports = {\n ...require('./jest.config.cjs'),\n testMatch: ['**/tests/integration/**/*.test.ts'],\n testTimeout: 60000\n};\n"
},
{
"type": "dir",
@@ -262,78 +267,122 @@
{
"type": "file",
"name": "index.ts",
- "template": "export * from './client'\nexport * from './config'\n"
+ "template": "export { create{connector|title}Connector } from './connector'\nexport type { {connector|title}Connector } from './types/connector'\nexport type { ConnectorConfig } from './types/config'\nexport type { HttpResponseEnvelope } from './types/envelopes'\n// Export all model types for external use\nexport type * from './models'\n// Observability exports\nexport { createLoggingHooks, createMetricsHooks, InMemoryMetricsSink, createInMemoryMetricsSink } from './observability'\n"
},
{
"type": "file",
- "name": "client.ts",
- "template": "export class Client {\n constructor(public config: Record) {}\n ping(): boolean { return true }\n}\n"
+ "name": "connector.ts",
+ "template": "import { ApiConnectorBase, type RateLimitOptions } from '@connector-factory/core'\nimport type { {connector|title}Connector } from './types/connector'\nimport type { ConnectorConfig } from './types/config'\nimport { withDerivedDefaults } from './config/defaults'\nimport { ConnectorError } from './types/errors'\nimport { build{resource|title}Domain } from './domains/{resource}'\nimport type { SendFn } from '@connector-factory/core'\n\nexport class {connector|title}ApiConnector extends ApiConnectorBase implements {connector|title}Connector {\n initialize(userConfig: ConnectorConfig) {\n const rateLimitOptions: RateLimitOptions = {\n onRateLimitSignal: (info) => {\n // Adaptive rate limiting based on server feedback\n if (this.config?.rateLimit?.adaptiveFromHeaders && this.limiter) {\n (this.limiter as any).updateFromResponse(info)\n }\n }\n }\n\n super.initialize(\n userConfig,\n withDerivedDefaults,\n ({ headers }: { headers: Record }) => {\n // Apply authentication\n if (this.config?.auth.type === 'bearer') {\n const token = this.config?.auth.bearer?.token\n if (!token) throw new ConnectorError({ message: 'Authentication failed – missing bearer token', code: 'AUTH_FAILED', source: 'auth', retryable: false })\n headers['Authorization'] = `Bearer ${token}`\n }\n },\n rateLimitOptions\n )\n }\n\n // Build domain delegates\n private get domain() {\n const sendLite: SendFn = async (args) => this.send(args)\n return {\n ...build{resource|title}Domain((args) => sendLite({ ...args, operation: args.operation ?? '{resource}' }))\n }\n }\n\n // {resource|title} methods\n list{resource|title} = (params?: { properties?: string[]; limit?: number; after?: string }) => this.domain.list{resource|title}(params)\n get{resource|title} = (params: { id: string; properties?: string[] }) => this.domain.get{resource|title}(params)\n stream{resource|title} = (params?: { properties?: string[]; pageSize?: number }) => this.domain.stream{resource|title}(params)\n get{resource|title}s = (params?: { properties?: string[]; pageSize?: number; maxItems?: number }) => this.domain.get{resource|title}s(params)\n}\n\nexport function create{connector|title}Connector(): {connector|title}Connector {\n return new {connector|title}ApiConnector()\n}\n"
},
{
"type": "file",
"name": "config.ts",
- "template": "export type ConnectorConfig = { apiKey: string }\n"
+ "template": "export type ConnectorConfig = {\n // Base configuration\n baseUrl?: string\n timeoutMs?: number\n userAgent?: string\n defaultHeaders?: Record\n defaultQueryParams?: Record\n\n // Authentication\n auth: {\n type: 'bearer' | 'apiKey' | 'basic'\n bearer?: { token: string }\n apiKey?: { key: string; header?: string }\n basic?: { username: string; password: string }\n }\n\n // Rate limiting\n rateLimit?: {\n requestsPerSecond?: number\n burstCapacity?: number\n concurrentRequests?: number\n adaptiveFromHeaders?: boolean\n }\n\n // Retry configuration\n retry?: {\n maxAttempts?: number\n initialDelayMs?: number\n maxDelayMs?: number\n backoffMultiplier?: number\n retryableStatusCodes?: number[]\n respectRetryAfter?: boolean\n }\n\n // Hooks\n hooks?: {\n beforeRequest?: Array<{ name: string; priority?: number; execute: (ctx: any) => Promise | void }>\n afterResponse?: Array<{ name: string; priority?: number; execute: (ctx: any) => Promise | void }>\n onError?: Array<{ name: string; priority?: number; execute: (ctx: any) => Promise | void }>\n onRetry?: Array<{ name: string; priority?: number; execute: (ctx: any) => Promise | void }>\n }\n}\n"
},
{
"type": "dir",
- "name": "auth",
+ "name": "domains",
"children": [
{
"type": "file",
- "name": "base.ts",
- "template": "export interface AuthStrategy { apply(headers: Record): Record }\n"
- },
- {
- "type": "file",
- "name": "apiKey.ts",
- "template": "import type { AuthStrategy } from './base'\nexport class ApiKeyAuth implements AuthStrategy {\n constructor(private apiKey: string) {}\n apply(headers: Record) { return { ...headers, Authorization: `Bearer ${this.apiKey}` } }\n}\n"
- },
+ "name": "{resource}.ts",
+ "template": "import type { SendFn } from '@connector-factory/core'\nimport { makeCrudDomain } from '../core/make-crud-domain'\nimport type { {resource|title}, {resource|title}sResponse, {resource|title}Response } from '../models/{resource}'\n\nexport function build{resource|title}Domain(send: SendFn) {\n const base = makeCrudDomain<{resource|title}, {resource|title}sResponse, {resource|title}Response>('/{resource}', send)\n return {\n list{resource|title}: base.list,\n get{resource|title}: base.get,\n stream{resource|title}: base.streamAll,\n get{resource|title}s: base.getAll\n }\n}\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "core",
+ "children": [
{
"type": "file",
- "name": "oauth2.ts",
- "template": "export {}\n"
+ "name": "make-crud-domain.ts",
+ "template": "import { paginateCursor, type SendFn } from '@connector-factory/core'\n\nexport function makeCrudDomain(objectPath: string, send: SendFn) {\n const api = {\n list: (params?: { properties?: string[]; limit?: number; after?: string }) => {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n if (params?.limit) query.limit = params.limit\n if (params?.after) query.after = params.after\n return send({ method: 'GET', path: objectPath, query })\n },\n get: (params: { id: string; properties?: string[] }) => {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n return send({ method: 'GET', path: `${objectPath}/${params.id}` as const, query })\n },\n streamAll: async function* (params?: { properties?: string[]; pageSize?: number }) {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n for await (const items of paginateCursor({ send, path: objectPath, query, pageSize: params?.pageSize })) {\n for (const item of items) yield item\n }\n },\n getAll: async (params?: { properties?: string[]; pageSize?: number; maxItems?: number }) => {\n const results: TObject[] = []\n for await (const item of api.streamAll({ properties: params?.properties, pageSize: params?.pageSize })) {\n results.push(item)\n if (params?.maxItems && results.length >= params.maxItems) break\n }\n return results\n }\n }\n return api\n}\n"
}
]
},
{
"type": "dir",
- "name": "lib",
+ "name": "types",
"children": [
{
"type": "file",
- "name": "paginate.ts",
- "template": "export type HttpResponseEnvelope = { data: T }\n\nexport type SendFn = (args: {\n method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';\n path: string;\n query?: Record;\n headers?: Record;\n body?: unknown;\n operation?: string;\n}) => Promise>\n\nexport async function* paginateCursor(params: {\n send: SendFn;\n path: string;\n query?: Record;\n pageSize?: number;\n extractItems?: (res: any) => T[];\n extractNextCursor?: (res: any) => string | undefined;\n}) {\n const extractItems = params.extractItems ?? ((res: any) => (res?.results ?? []) as T[])\n const extractNext = params.extractNextCursor ?? ((res: any) => res?.paging?.next?.after as string | undefined)\n let after: string | undefined = params.query?.after as string | undefined\n const limit = params.pageSize ?? (params.query?.limit as number | undefined) ?? 100\n // eslint-disable-next-line no-constant-condition\n while (true) {\n const res = await params.send({ method: 'GET', path: params.path, query: { ...(params.query ?? {}), limit, after }, operation: 'paginate' })\n const items = extractItems(res.data)\n yield items\n const next = extractNext(res.data)\n if (!next) break\n after = next\n }\n}\n"
+ "name": "connector.ts",
+ "template": "import type { ConnectorConfig } from './config'\nimport type { {resource|title} } from '../models/{resource}'\n\nexport interface {connector|title}Connector {\n initialize(config: ConnectorConfig): void\n connect(): Promise\n disconnect(): Promise\n isConnected(): boolean\n\n // {resource|title} operations\n list{resource|title}(params?: { properties?: string[]; limit?: number; after?: string }): Promise\n get{resource|title}(params: { id: string; properties?: string[] }): Promise\n stream{resource|title}(params?: { properties?: string[]; pageSize?: number }): AsyncGenerator<{resource|title}, void, unknown>\n get{resource|title}s(params?: { properties?: string[]; pageSize?: number; maxItems?: number }): Promise<{resource|title}[]>\n}\n"
},
{
"type": "file",
- "name": "make-resource.ts",
- "template": "import { paginateCursor, type SendFn } from './paginate'\n\nexport function makeCrudResource(objectPath: string, send: SendFn) {\n const api = {\n list: (params?: { properties?: string[]; limit?: number; after?: string }) => {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n if (params?.limit) query.limit = params.limit\n if (params?.after) query.after = params.after\n return send({ method: 'GET', path: objectPath, query })\n },\n get: (params: { id: string; properties?: string[] }) => {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n return send({ method: 'GET', path: `${objectPath}/${params.id}`, query })\n },\n streamAll: async function* (params?: { properties?: string[]; pageSize?: number }) {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n for await (const items of paginateCursor({ send, path: objectPath, query, pageSize: params?.pageSize })) {\n for (const item of items) yield item\n }\n },\n getAll: async (params?: { properties?: string[]; pageSize?: number; maxItems?: number }) => {\n const results: TItem[] = []\n for await (const item of api.streamAll({ properties: params?.properties, pageSize: params?.pageSize })) {\n results.push(item)\n if (params?.maxItems && results.length >= params.maxItems) break\n }\n return results\n },\n }\n return api\n}\n"
+ "name": "envelopes.ts",
+ "template": "export type HttpResponseEnvelope = {\n data: T\n status?: number\n headers?: Record\n meta?: {\n timestamp: number\n durationMs: number\n retryCount: number\n requestId?: string\n rateLimit?: {\n limit: number\n remaining: number\n reset: number\n resetDate?: string\n }\n }\n}\n"
},
{
"type": "file",
- "name": "hooks.ts",
- "template": "export type RequestOptions = {\n method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE'\n path: string\n query?: Record\n headers?: Record\n body?: unknown\n timeoutMs?: number\n operation?: string\n}\n\nexport type ResponseMeta = {\n timestamp: number\n durationMs: number\n retryCount: number\n requestId?: string\n}\n\nexport type ResponseEnvelope = {\n data: T\n status?: number\n headers?: Record\n meta?: ResponseMeta\n}\n\nexport type HookContext = {\n type: 'beforeRequest' | 'afterResponse' | 'onError' | 'onRetry'\n request?: RequestOptions\n response?: ResponseEnvelope\n error?: unknown\n metadata?: Record\n attempt?: number\n}\n\nexport type Hook = (ctx: HookContext) => void | Promise\n\nexport type Hooks = {\n beforeRequest?: Hook[]\n afterResponse?: Hook[]\n onError?: Hook[]\n onRetry?: Hook[]\n}\n\nexport async function runHooks(hooks: Hook[] | undefined, ctx: HookContext): Promise {\n for (const hook of hooks ?? []) {\n // eslint-disable-next-line no-await-in-loop\n await hook(ctx)\n }\n}\n"
+ "name": "errors.ts",
+ "template": "export type ErrorCode = 'NETWORK_ERROR' | 'TIMEOUT' | 'AUTH_FAILED' | 'RATE_LIMITED' | 'INVALID_REQUEST' | 'SERVER_ERROR' | 'PARSING_ERROR' | 'VALIDATION_ERROR' | 'CANCELLED' | 'UNSUPPORTED' | 'NOT_INITIALIZED'\n\nexport type ErrorSource = 'transport' | 'auth' | 'rateLimit' | 'deserialize' | 'userHook' | 'application' | 'unknown'\n\nexport interface ConnectorErrorOptions {\n message: string\n code: ErrorCode\n statusCode?: number\n details?: any\n retryable: boolean\n requestId?: string\n source: ErrorSource\n}\n\nexport class ConnectorError extends Error {\n code: ErrorCode\n statusCode?: number\n retryable: boolean\n requestId?: string\n source: ErrorSource\n details?: any\n\n constructor(options: ConnectorErrorOptions) {\n super(options.message)\n this.name = 'ConnectorError'\n this.code = options.code\n this.statusCode = options.statusCode\n this.retryable = options.retryable\n this.requestId = options.requestId\n this.source = options.source\n this.details = options.details\n }\n}\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "models",
+ "children": [
+ {
+ "type": "file",
+ "name": "index.ts",
+ "template": "export * from './{resource}'\n"
},
+ {
+ "type": "dir",
+ "name": "{resource}",
+ "children": [
+ {
+ "type": "file",
+ "name": "index.ts",
+ "template": "export * from './{resource}'\nexport * from './{resource}-api-contracts'\n"
+ },
+ {
+ "type": "file",
+ "name": "{resource}.ts",
+ "template": "export interface {resource|title} {\n id: string\n // Add your resource-specific properties here\n [key: string]: unknown\n}\n"
+ },
+ {
+ "type": "file",
+ "name": "{resource}-api-contracts.ts",
+ "template": "import type { {resource|title} } from './{resource}'\n\nexport interface {resource|title}sResponse {\n results: {resource|title}[]\n paging?: {\n next?: {\n after?: string\n }\n }\n}\n\nexport interface {resource|title}Response {\n data: {resource|title}\n}\n"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "config",
+ "children": [
{
"type": "file",
- "name": "send.ts",
- "template": "import type { Hooks, RequestOptions, ResponseEnvelope } from './hooks'\nimport { runHooks } from './hooks'\n\nexport type DoRequest = (req: RequestOptions) => Promise>\n\nexport type RetryConfig = {\n maxAttempts?: number\n initialDelayMs?: number\n maxDelayMs?: number\n backoffMultiplier?: number\n retryableStatusCodes?: number[]\n respectRetryAfter?: boolean\n}\n\nfunction sleep(ms: number): Promise {\n return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\nfunction calcDelay(attempt: number, cfg: Required>): number {\n const exp = cfg.initialDelayMs * Math.pow(cfg.backoffMultiplier, attempt - 1)\n const bounded = Math.min(exp, cfg.maxDelayMs)\n const jitter = bounded * (0.5 + Math.random() * 0.5)\n return Math.floor(jitter)\n}\n\nexport function createSend({ doRequest, hooks, retry }: { doRequest: DoRequest; hooks?: Hooks; retry?: RetryConfig }) {\n const retryCfg: Required = {\n maxAttempts: retry?.maxAttempts ?? 3,\n initialDelayMs: retry?.initialDelayMs ?? 1000,\n maxDelayMs: retry?.maxDelayMs ?? 30000,\n backoffMultiplier: retry?.backoffMultiplier ?? 2,\n retryableStatusCodes: retry?.retryableStatusCodes ?? [408, 425, 429, 500, 502, 503, 504],\n respectRetryAfter: retry?.respectRetryAfter ?? true,\n } as Required\n\n async function send(req: RequestOptions): Promise> {\n const startedAt = Date.now()\n let lastError: unknown\n for (let attempt = 1; attempt <= retryCfg.maxAttempts; attempt++) {\n try {\n await runHooks(hooks?.beforeRequest, { type: 'beforeRequest', request: req, attempt })\n const res = await doRequest(req)\n await runHooks(hooks?.afterResponse, { type: 'afterResponse', request: req, response: res, attempt })\n const status = res.status ?? 200\n if (!retryCfg.retryableStatusCodes.includes(status)) {\n res.meta = { ...(res.meta ?? {}), timestamp: Date.now(), durationMs: Date.now() - startedAt, retryCount: attempt - 1 }\n return res\n }\n lastError = new Error(`Retryable status: ${status}`)\n } catch (err) {\n lastError = err\n await runHooks(hooks?.onError, { type: 'onError', request: req, error: err, attempt })\n }\n if (attempt < retryCfg.maxAttempts) {\n await runHooks(hooks?.onRetry, { type: 'onRetry', request: req, error: lastError, attempt })\n const delay = calcDelay(attempt, { initialDelayMs: retryCfg.initialDelayMs, maxDelayMs: retryCfg.maxDelayMs, backoffMultiplier: retryCfg.backoffMultiplier })\n await sleep(delay)\n continue\n }\n break\n }\n throw lastError ?? new Error('Request failed')\n }\n\n return send\n}\n"
+ "name": "defaults.ts",
+ "template": "import type { ConnectorConfig } from '../types/config'\n\nexport function withDerivedDefaults(userConfig: ConnectorConfig): ConnectorConfig {\n return {\n ...userConfig,\n baseUrl: userConfig.baseUrl ?? 'https://api.example.com',\n timeoutMs: userConfig.timeoutMs ?? 30000,\n userAgent: userConfig.userAgent ?? '{connector} Connector v1.0.0',\n rateLimit: {\n requestsPerSecond: 10,\n burstCapacity: 20,\n concurrentRequests: 5,\n adaptiveFromHeaders: true,\n ...userConfig.rateLimit\n },\n retry: {\n maxAttempts: 3,\n initialDelayMs: 1000,\n maxDelayMs: 30000,\n backoffMultiplier: 2,\n retryableStatusCodes: [408, 425, 429, 500, 502, 503, 504],\n respectRetryAfter: true,\n ...userConfig.retry\n }\n }\n}\n"
}
]
},
{
"type": "dir",
- "name": "{resource}",
+ "name": "observability",
"children": [
{
"type": "file",
"name": "index.ts",
- "template": "import { makeCrudResource } from '../lib/make-resource'\nimport type { SendFn } from '../lib/paginate'\nimport type { Model } from './model'\n\nexport const createResource = (send: SendFn) => makeCrudResource('/{resource}', send)\n"
+ "template": "export { createLoggingHooks } from './logging-hooks'\nexport { createMetricsHooks, InMemoryMetricsSink, createInMemoryMetricsSink } from './metrics-hooks'\n"
+ },
+ {
+ "type": "file",
+ "name": "logging-hooks.ts",
+ "template": "export function createLoggingHooks() {\n return {\n beforeRequest: [{\n name: 'request-logger',\n priority: 100,\n execute: (ctx: any) => {\n console.log(`[${ctx.type}] Making request to ${ctx.request?.path}`)\n }\n }],\n afterResponse: [{\n name: 'response-logger',\n priority: 100,\n execute: (ctx: any) => {\n console.log(`[${ctx.type}] Response received with status ${ctx.response?.status}`)\n }\n }]\n }\n}\n"
},
{
"type": "file",
- "name": "model.ts",
- "template": "export interface Model {\n // Define your resource model fields\n [key: string]: unknown\n}\n"
+ "name": "metrics-hooks.ts",
+ "template": "export interface MetricsSink {\n increment(name: string, tags?: Record): void\n gauge(name: string, value: number, tags?: Record): void\n histogram(name: string, value: number, tags?: Record): void\n}\n\nexport class InMemoryMetricsSink implements MetricsSink {\n private metrics = new Map()\n\n increment(name: string, tags?: Record): void {\n // Implementation for increment\n }\n\n gauge(name: string, value: number, tags?: Record): void {\n // Implementation for gauge\n }\n\n histogram(name: string, value: number, tags?: Record): void {\n // Implementation for histogram\n }\n\n getMetrics() {\n return this.metrics\n }\n}\n\nexport function createInMemoryMetricsSink(): InMemoryMetricsSink {\n return new InMemoryMetricsSink()\n}\n\nexport function createMetricsHooks(sink: MetricsSink) {\n return {\n beforeRequest: [{\n name: 'metrics-collector',\n priority: 100,\n execute: (ctx: any) => {\n sink.increment('requests.total', { operation: ctx.operation })\n }\n }]\n }\n}\n"
}
]
}
@@ -344,9 +393,31 @@
"name": "tests",
"children": [
{
- "type": "file",
- "name": "client.test.ts",
- "template": "import { describe, it, expect } from 'vitest'\nimport { Client } from '../src/client'\n\ndescribe('client', () => {\n it('ping', () => {\n expect(new Client({}).ping()).toBe(true)\n })\n})\n"
+ "type": "dir",
+ "name": "unit",
+ "children": [
+ {
+ "type": "file",
+ "name": "connector.test.ts",
+ "template": "import { create{connector|title}Connector } from '../src/connector'\nimport type { ConnectorConfig } from '../src/types/config'\n\ndescribe('{connector|title}Connector', () => {\n let connector: ReturnType\n let config: ConnectorConfig\n\n beforeEach(() => {\n connector = create{connector|title}Connector()\n config = {\n auth: {\n type: 'bearer',\n bearer: { token: 'test-token' }\n }\n }\n })\n\n it('should initialize successfully', () => {\n expect(() => connector.initialize(config)).not.toThrow()\n })\n\n it('should handle connection lifecycle', async () => {\n connector.initialize(config)\n expect(connector.isConnected()).toBe(false)\n await connector.connect()\n expect(connector.isConnected()).toBe(true)\n await connector.disconnect()\n expect(connector.isConnected()).toBe(false)\n })\n})\n"
+ },
+ {
+ "type": "file",
+ "name": "{resource}.test.ts",
+ "template": "import { create{connector|title}Connector } from '../src/connector'\nimport type { ConnectorConfig } from '../src/types/config'\n\ndescribe('{resource|title} operations', () => {\n let connector: ReturnType\n let config: ConnectorConfig\n\n beforeEach(() => {\n connector = create{connector|title}Connector()\n config = {\n auth: {\n type: 'bearer',\n bearer: { token: 'test-token' }\n }\n }\n connector.initialize(config)\n })\n\n it('should list {resource}', async () => {\n // Mock test - implement actual testing logic\n expect(connector.list{resource|title}).toBeDefined()\n })\n\n it('should get single {resource}', async () => {\n // Mock test - implement actual testing logic\n expect(connector.get{resource|title}).toBeDefined()\n })\n})\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "integration",
+ "children": [
+ {
+ "type": "file",
+ "name": "{resource}.integration.test.ts",
+ "template": "import { create{connector|title}Connector } from '../src/connector'\nimport type { ConnectorConfig } from '../src/types/config'\n\n// Integration tests - require real API credentials\ndescribe.skip('{resource|title} integration tests', () => {\n let connector: ReturnType\n let config: ConnectorConfig\n\n beforeAll(() => {\n if (!process.env.API_TOKEN) {\n throw new Error('API_TOKEN environment variable required for integration tests')\n }\n\n connector = create{connector|title}Connector()\n config = {\n auth: {\n type: 'bearer',\n bearer: { token: process.env.API_TOKEN }\n }\n }\n connector.initialize(config)\n })\n\n it('should connect to real API', async () => {\n await connector.connect()\n expect(connector.isConnected()).toBe(true)\n })\n\n it('should list {resource} from real API', async () => {\n await connector.connect()\n const result = await connector.list{resource|title}({ limit: 1 })\n expect(result).toBeDefined()\n })\n})\n"
+ }
+ ]
}
]
},
diff --git a/connector-registry/hubspot/v3/514-labs/_meta/connector.json b/connector-registry/hubspot/v3/514-labs/_meta/connector.json
index a862c4cf..05300d62 100644
--- a/connector-registry/hubspot/v3/514-labs/_meta/connector.json
+++ b/connector-registry/hubspot/v3/514-labs/_meta/connector.json
@@ -18,7 +18,7 @@
"maintainers": [],
"issues": {
"typescript": {
- "data-api": "https://github.com/514-labs/factory/issues/26"
+ "data-api": "https://github.com/514-labs/registry/issues/26"
}
}
}
diff --git a/pipeline-registry/_scaffold/python.json b/pipeline-registry/_scaffold/python.json
index 0742bd0f..c7988d47 100644
--- a/pipeline-registry/_scaffold/python.json
+++ b/pipeline-registry/_scaffold/python.json
@@ -65,7 +65,7 @@
{
"type": "file",
"name": "pipeline.json",
- "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/pipeline.schema.json\",\n \"identifier\": \"{pipeline}\",\n \"name\": \"{pipeline}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"version\": \"{version}\",\n \"language\": \"python\",\n \"implementation\": \"{implementation}\",\n \"description\": \"\",\n \"tags\": [],\n \"schedule\": { \"cron\": \"0 * * * *\", \"timezone\": \"UTC\" },\n \"source\": { \"type\": \"connector\", \"connector\": { \"name\": \"\", \"version\": \"\", \"author\": \"\" }, \"stream\": \"\" },\n \"systems\": [],\n \"transformations\": [],\n \"destination\": { \"system\": \"clickhouse\", \"database\": \"\", \"table\": \"\" },\n \"lineage\": { \"nodes\": [], \"edges\": [] },\n \"maintainers\": [],\n \"registryUrl\": \"https://github.com/514-labs/factory/tree/main/pipeline-registry/{pipeline}/{version}/{author}/python/{implementation}\"\n}\n"
+ "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/pipeline.schema.json\",\n \"identifier\": \"{pipeline}\",\n \"name\": \"{pipeline}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"version\": \"{version}\",\n \"language\": \"python\",\n \"implementation\": \"{implementation}\",\n \"description\": \"\",\n \"tags\": [],\n \"schedule\": { \"cron\": \"0 * * * *\", \"timezone\": \"UTC\" },\n \"source\": { \"type\": \"connector\", \"connector\": { \"name\": \"\", \"version\": \"\", \"author\": \"\" }, \"stream\": \"\" },\n \"systems\": [],\n \"transformations\": [],\n \"destination\": { \"system\": \"clickhouse\", \"database\": \"\", \"table\": \"\" },\n \"lineage\": { \"nodes\": [], \"edges\": [] },\n \"maintainers\": [],\n \"registryUrl\": \"https://github.com/514-labs/registry/tree/main/pipeline-registry/{pipeline}/{version}/{author}/python/{implementation}\"\n}\n"
},
{
"type": "file",
diff --git a/pipeline-registry/_scaffold/typescript.json b/pipeline-registry/_scaffold/typescript.json
index ab9544b4..36bd3e5b 100644
--- a/pipeline-registry/_scaffold/typescript.json
+++ b/pipeline-registry/_scaffold/typescript.json
@@ -65,7 +65,7 @@
{
"type": "file",
"name": "pipeline.json",
- "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/pipeline.schema.json\",\n \"identifier\": \"{pipeline}\",\n \"name\": \"{pipeline}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"version\": \"{version}\",\n \"language\": \"typescript\",\n \"implementation\": \"{implementation}\",\n \"description\": \"\",\n \"tags\": [],\n \"schedule\": { \"cron\": \"0 * * * *\", \"timezone\": \"UTC\" },\n \"source\": { \"type\": \"connector\", \"connector\": { \"name\": \"\", \"version\": \"\", \"author\": \"\" }, \"stream\": \"\" },\n \"systems\": [],\n \"transformations\": [],\n \"destination\": { \"system\": \"clickhouse\", \"database\": \"\", \"table\": \"\" },\n \"lineage\": { \"nodes\": [], \"edges\": [] },\n \"maintainers\": [],\n \"registryUrl\": \"https://github.com/514-labs/factory/tree/main/pipeline-registry/{pipeline}/{version}/{author}/typescript/{implementation}\"\n}\n"
+ "template": "{\n \"$schema\": \"https://schemas.connector-factory.dev/pipeline.schema.json\",\n \"identifier\": \"{pipeline}\",\n \"name\": \"{pipeline}\",\n \"author\": \"{author}\",\n \"authorType\": \"organization\",\n \"version\": \"{version}\",\n \"language\": \"typescript\",\n \"implementation\": \"{implementation}\",\n \"description\": \"\",\n \"tags\": [],\n \"schedule\": { \"cron\": \"0 * * * *\", \"timezone\": \"UTC\" },\n \"source\": { \"type\": \"connector\", \"connector\": { \"name\": \"\", \"version\": \"\", \"author\": \"\" }, \"stream\": \"\" },\n \"systems\": [],\n \"transformations\": [],\n \"destination\": { \"system\": \"clickhouse\", \"database\": \"\", \"table\": \"\" },\n \"lineage\": { \"nodes\": [], \"edges\": [] },\n \"maintainers\": [],\n \"registryUrl\": \"https://github.com/514-labs/registry/tree/main/pipeline-registry/{pipeline}/{version}/{author}/typescript/{implementation}\"\n}\n"
},
{
"type": "file",
@@ -115,7 +115,7 @@
},
{
"type": "file",
- "name": ".env.example",
+ "name": "ENV.EXAMPLE",
"template": "# Example environment variables for the pipeline implementation\nAPI_KEY=\n"
},
{
@@ -126,7 +126,7 @@
{
"type": "file",
"name": "package.json",
- "template": "{\n \"name\": \"{packageName}\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"type\": \"module\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts --dts --format esm,cjs\",\n \"test\": \"vitest run\"\n },\n \"engines\": {\"node\": \">=20\"},\n \"dependencies\": {},\n \"devDependencies\": {\n \"tsup\": \"^8.0.0\",\n \"typescript\": \"^5.4.0\",\n \"vitest\": \"^1.6.0\"\n }\n}\n"
+ "template": "{\n \"name\": \"{packageName}\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"cd app/{resource}; pnpm run build; cd ../..\",\n \"test\": \"vitest run\",\n \"dev\": \"moose-cli dev\",\n \"moose\": \"moose-cli\"\n },\n \"engines\": {\"node\": \">=20\"},\n \"dependencies\": {\n \"@514labs/moose-lib\": \"latest\",\n \"dotenv\": \"^16.4.5\",\n \"typia\": \"^9.6.1\"\n },\n \"devDependencies\": {\n \"@514labs/moose-cli\": \"latest\",\n \"tsup\": \"^8.0.0\",\n \"typescript\": \"^5.4.0\",\n \"vitest\": \"^3.2.4\"\n }\n}\n"
},
{
"type": "file",
@@ -232,7 +232,7 @@
},
{
"type": "dir",
- "name": "src",
+ "name": "app",
"children": [
{
"type": "file",
@@ -251,49 +251,76 @@
},
{
"type": "dir",
- "name": "lib",
+ "name": "extract",
"children": [
{
"type": "file",
- "name": "paginate.ts",
- "template": "export type HttpResponseEnvelope = { data: T }\n\nexport type SendFn = (args: {\n method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';\n path: string;\n query?: Record;\n headers?: Record;\n body?: unknown;\n operation?: string;\n}) => Promise>\n\nexport async function* paginateCursor(params: {\n send: SendFn;\n path: string;\n query?: Record;\n pageSize?: number;\n extractItems?: (res: any) => T[];\n extractNextCursor?: (res: any) => string | undefined;\n}) {\n const extractItems = params.extractItems ?? ((res: any) => (res?.results ?? []) as T[])\n const extractNext = params.extractNextCursor ?? ((res: any) => res?.paging?.next?.after as string | undefined)\n let after: string | undefined = params.query?.after as string | undefined\n const limit = params.pageSize ?? (params.query?.limit as number | undefined) ?? 100\n // eslint-disable-next-line no-constant-condition\n while (true) {\n const res = await params.send({ method: 'GET', path: params.path, query: { ...(params.query ?? {}), limit, after }, operation: 'paginate' })\n const items = extractItems(res.data)\n yield items\n const next = extractNext(res.data)\n if (!next) break\n after = next\n }\n}\n"
- },
- {
- "type": "file",
- "name": "make-resource.ts",
- "template": "import { paginateCursor, type SendFn } from './paginate'\n\n/**\n * makeCrudResource\n * Creates a CRUD surface for a REST resource at `objectPath`.\n */\nexport function makeCrudResource(objectPath: string, send: SendFn) {\n const api = {\n list: (params?: { properties?: string[]; limit?: number; after?: string }) => {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n if (params?.limit) query.limit = params.limit\n if (params?.after) query.after = params.after\n return send({ method: 'GET', path: objectPath, query })\n },\n get: (params: { id: string; properties?: string[] }) => {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n return send({ method: 'GET', path: `${objectPath}/${params.id}`, query })\n },\n streamAll: async function* (params?: { properties?: string[]; pageSize?: number }) {\n const query: Record = {}\n if (params?.properties?.length) query.properties = params.properties.join(',')\n for await (const items of paginateCursor({ send, path: objectPath, query, pageSize: params?.pageSize })) {\n for (const item of items) yield item\n }\n },\n getAll: async (params?: { properties?: string[]; pageSize?: number; maxItems?: number }) => {\n const results: TItem[] = []\n for await (const item of api.streamAll({ properties: params?.properties, pageSize: params?.pageSize })) {\n results.push(item)\n if (params?.maxItems && results.length >= params.maxItems) break\n }\n return results\n },\n }\n return api\n}\n"
- },
+ "name": "baseExtractor.ts",
+ "template": "export abstract class BaseExtractor {\n abstract run(): Promise\n}\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "transform",
+ "children": [
{
"type": "file",
- "name": "hooks.ts",
- "template": "export type RequestOptions = {\n method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE'\n path: string\n query?: Record\n headers?: Record\n body?: unknown\n timeoutMs?: number\n operation?: string\n}\n\nexport type ResponseMeta = {\n timestamp: number\n durationMs: number\n retryCount: number\n requestId?: string\n}\n\nexport type ResponseEnvelope = {\n data: T\n status?: number\n headers?: Record\n meta?: ResponseMeta\n}\n\nexport type HookContext = {\n type: 'beforeRequest' | 'afterResponse' | 'onError' | 'onRetry'\n request?: RequestOptions\n response?: ResponseEnvelope\n error?: unknown\n metadata?: Record\n attempt?: number\n}\n\nexport type Hook = (ctx: HookContext) => void | Promise\n\nexport type Hooks = {\n beforeRequest?: Hook[]\n afterResponse?: Hook[]\n onError?: Hook[]\n onRetry?: Hook[]\n}\n\nexport async function runHooks(hooks: Hook[] | undefined, ctx: HookContext): Promise {\n for (const hook of hooks ?? []) {\n // eslint-disable-next-line no-await-in-loop\n await hook(ctx)\n }\n}\n"
- },
+ "name": "baseTransformer.ts",
+ "template": "export abstract class BaseTransformer {\n abstract run(): Promise\n}\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "load",
+ "children": [
{
"type": "file",
- "name": "send.ts",
- "template": "import type { Hooks, RequestOptions, ResponseEnvelope } from './hooks'\nimport { runHooks } from './hooks'\n\nexport type DoRequest = (req: RequestOptions) => Promise>\n\nexport type RetryConfig = {\n maxAttempts?: number\n initialDelayMs?: number\n maxDelayMs?: number\n backoffMultiplier?: number\n retryableStatusCodes?: number[]\n respectRetryAfter?: boolean\n}\n\nfunction sleep(ms: number): Promise {\n return new Promise((resolve) => setTimeout(resolve, ms))\n}\n\nfunction calcDelay(attempt: number, cfg: Required>): number {\n const exp = cfg.initialDelayMs * Math.pow(cfg.backoffMultiplier, attempt - 1)\n const bounded = Math.min(exp, cfg.maxDelayMs)\n const jitter = bounded * (0.5 + Math.random() * 0.5)\n return Math.floor(jitter)\n}\n\nexport function createSend({ doRequest, hooks, retry }: { doRequest: DoRequest; hooks?: Hooks; retry?: RetryConfig }) {\n const retryCfg: Required = {\n maxAttempts: retry?.maxAttempts ?? 3,\n initialDelayMs: retry?.initialDelayMs ?? 1000,\n maxDelayMs: retry?.maxDelayMs ?? 30000,\n backoffMultiplier: retry?.backoffMultiplier ?? 2,\n retryableStatusCodes: retry?.retryableStatusCodes ?? [408, 425, 429, 500, 502, 503, 504],\n respectRetryAfter: retry?.respectRetryAfter ?? true,\n } as Required\n\n async function send(req: RequestOptions): Promise> {\n const startedAt = Date.now()\n let lastError: unknown\n for (let attempt = 1; attempt <= retryCfg.maxAttempts; attempt++) {\n try {\n await runHooks(hooks?.beforeRequest, { type: 'beforeRequest', request: req, attempt })\n const res = await doRequest(req)\n await runHooks(hooks?.afterResponse, { type: 'afterResponse', request: req, response: res, attempt })\n const status = res.status ?? 200\n if (!retryCfg.retryableStatusCodes.includes(status)) {\n res.meta = { ...(res.meta ?? {}), timestamp: Date.now(), durationMs: Date.now() - startedAt, retryCount: attempt - 1 }\n return res\n }\n lastError = new Error(`Retryable status: ${status}`)\n } catch (err) {\n lastError = err\n await runHooks(hooks?.onError, { type: 'onError', request: req, error: err, attempt })\n }\n if (attempt < retryCfg.maxAttempts) {\n await runHooks(hooks?.onRetry, { type: 'onRetry', request: req, error: lastError, attempt })\n const delay = calcDelay(attempt, { initialDelayMs: retryCfg.initialDelayMs, maxDelayMs: retryCfg.maxDelayMs, backoffMultiplier: retryCfg.backoffMultiplier })\n await sleep(delay)\n continue\n }\n break\n }\n throw lastError ?? new Error('Request failed')\n }\n\n return send\n}\n"
+ "name": "baseLoader.ts",
+ "template": "export abstract class BaseLoader {\n abstract run(): Promise\n}\n"
}
]
},
{
"type": "dir",
- "name": "{resource}",
+ "name": "ingest",
"children": [
{
"type": "file",
- "name": "index.ts",
- "template": "import { makeCrudResource } from '../lib/make-resource'\nimport type { SendFn } from '../lib/paginate'\nimport type { Model } from './model'\n\nexport const createResource = (send: SendFn) => makeCrudResource('/{resource}', send)\n"
+ "name": "{resource}Models.ts",
+ "template": "// Data models for {resource} ingest\nexport interface {resource}Record {\n id: string\n timestamp: number\n}\n"
},
{
"type": "file",
- "name": "model.ts",
- "template": "export interface Model {\n // Define your resource model fields\n [key: string]: unknown\n}\n"
+ "name": "{resource}Transforms.ts",
+ "template": "// Transform functions for {resource} data\nexport function transform{resource}Data(data: any) {\n return data\n}\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "apis",
+ "children": [
+ {
+ "type": "file",
+ "name": "{resource}Workflow.ts",
+ "template": "// Workflow definitions for {resource} processing\nexport const {resource}WorkflowTrigger = {\n // trigger logic\n}\n"
}
]
}
]
},
-
+ {
+ "type": "file",
+ "name": "aurora.config.toml",
+ "template": "# Aurora configuration for {pipeline} pipeline\n[aurora]\nname = \"{pipeline}\"\nversion = \"{version}\"\n"
+ },
+ {
+ "type": "file",
+ "name": "moose.config.toml",
+ "template": "# Moose configuration for {pipeline} pipeline\n[moose]\nname = \"{pipeline}\"\nversion = \"{version}\"\n"
+ },
{
"type": "dir",
"name": "tests",
@@ -301,7 +328,18 @@
{
"type": "file",
"name": "runner.test.ts",
- "template": "import { describe, it, expect } from 'vitest'\nimport { PipelineRunner } from '../src/runner'\n\ndescribe('PipelineRunner', () => {\n it('ping', () => {\n expect(new PipelineRunner({}).ping()).toBe(true)\n })\n})\n"
+ "template": "import { describe, it, expect } from 'vitest'\nimport { PipelineRunner } from '../app/runner'\n\ndescribe('PipelineRunner', () => {\n it('ping', () => {\n expect(new PipelineRunner({}).ping()).toBe(true)\n })\n})\n"
+ }
+ ]
+ },
+ {
+ "type": "dir",
+ "name": "scripts",
+ "children": [
+ {
+ "type": "file",
+ "name": "initial-setup.sh",
+ "template": "#!/bin/bash\n# Initial setup script for {pipeline} pipeline\necho \"Setting up {pipeline} pipeline...\"\n"
}
]
},
diff --git a/pipeline-registry/google-analytics-to-clickhouse/_meta/pipeline.json b/pipeline-registry/google-analytics-to-clickhouse/_meta/pipeline.json
index b5a9a082..d6219aa1 100644
--- a/pipeline-registry/google-analytics-to-clickhouse/_meta/pipeline.json
+++ b/pipeline-registry/google-analytics-to-clickhouse/_meta/pipeline.json
@@ -5,5 +5,5 @@
"tags": ["google-analytics", "ga4", "clickhouse"],
"description": "Ingest GA4 events into ClickHouse.",
"homepage": "",
- "registryUrl": "https://github.com/514-labs/factory/tree/main/pipeline-registry/google-analytics-to-clickhouse"
+ "registryUrl": "https://github.com/514-labs/registry/tree/main/pipeline-registry/google-analytics-to-clickhouse"
}
diff --git a/pipeline-registry/hubspot-to-clickhouse/_meta/pipeline.json b/pipeline-registry/hubspot-to-clickhouse/_meta/pipeline.json
index e17e298b..18a9673b 100644
--- a/pipeline-registry/hubspot-to-clickhouse/_meta/pipeline.json
+++ b/pipeline-registry/hubspot-to-clickhouse/_meta/pipeline.json
@@ -5,6 +5,6 @@
"tags": ["hubspot", "clickhouse"],
"description": "Ingest HubSpot data into ClickHouse.",
"homepage": "",
- "registryUrl": "https://github.com/514-labs/factory/tree/main/pipeline-registry/hubspot-to-clickhouse"
+ "registryUrl": "https://github.com/514-labs/registry/tree/main/pipeline-registry/hubspot-to-clickhouse"
}