Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 160 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@

# Stage 1: Full monorepo context for dependency resolution
FROM node:20-bookworm-slim AS monorepo-base

# This is to remove the notice to update NPM that will break the output from STDOUT
RUN npm config set update-notifier false

# Install alternative package managers globally
RUN npm install -g pnpm@latest

# Set working directory to monorepo root
WORKDIR /monorepo

# Copy workspace configuration files
COPY pnpm-workspace.yaml ./
COPY package.json ./
COPY .npmrc ./
COPY pnpm-lock.yaml ./

# Copy workspace package directories (will be replaced with actual patterns)
COPY apps ./apps
COPY packages ./packages
COPY examples ./examples
COPY packages/cli ./packages/cli

# Install all dependencies from monorepo root
RUN pnpm install --frozen-lockfile

# Stage 2: Production image
FROM node:20-bookworm-slim

# This is to remove the notice to update NPM that will break the output from STDOUT
RUN npm config set update-notifier false

# Install alternative package managers globally
RUN npm install -g pnpm@latest

ARG DEBIAN_FRONTEND=noninteractive

# Update the package lists for upgrades for security purposes
RUN apt-get update && apt-get upgrade -y

# Install ca-certificates, tail and locales package
RUN apt-get install -y ca-certificates locales coreutils curl && update-ca-certificates

# moose depends on libc 2.40+, not available in stable
# This uses unstable, but unpinned because they delete older versions
RUN echo "deb http://deb.debian.org/debian/ unstable main" >> /etc/apt/sources.list \
&& apt-get update \
&& apt-get install -y libc6 \
&& apt-get clean && rm -rf /var/lib/apt/lists/*

# Generate locale files
RUN locale-gen en_US.UTF-8
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
ENV TZ=UTC
ENV DOCKER_IMAGE=true

# Install Moose
ARG FRAMEWORK_VERSION="0.0.0"
ARG DOWNLOAD_URL
RUN echo "DOWNLOAD_URL: ${DOWNLOAD_URL}"
RUN ldd --version
RUN curl -Lo /usr/local/bin/moose ${DOWNLOAD_URL}
RUN chmod +x /usr/local/bin/moose

RUN moose --version

# Setup healthcheck
HEALTHCHECK --interval=30s --timeout=3s \
CMD curl -f http://localhost:4000/health || exit 1

# Sets up non-root user using 1001 because node creates a user with 1000
RUN groupadd --gid 1001 moose \
&& useradd --uid 1001 --gid moose --shell /bin/bash --create-home moose

# Set the working directory inside the container
WORKDIR /application

# Ensure application directory is owned by moose user
RUN chown -R moose:moose /application

# Placeholder for the language specific copy package file copy

# Copy application files from monorepo stage
COPY --from=monorepo-base --chown=moose:moose /monorepo/pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api/app ./app
COPY --from=monorepo-base --chown=moose:moose /monorepo/pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api/package.json ./package.json
COPY --from=monorepo-base --chown=moose:moose /monorepo/pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api/tsconfig.json ./tsconfig.json

# Use pnpm deploy from workspace context to create clean production dependencies
USER root:root
WORKDIR /temp-monorepo
COPY --from=monorepo-base /monorepo/pnpm-workspace.yaml ./
COPY --from=monorepo-base /monorepo/pnpm-lock.yaml ./
COPY --from=monorepo-base /monorepo/pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api ./pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api
# Copy all workspace directories that exist
COPY --from=monorepo-base /monorepo/apps ./apps
COPY --from=monorepo-base /monorepo/packages ./packages
COPY --from=monorepo-base /monorepo/examples ./examples
COPY --from=monorepo-base /monorepo/packages/cli ./packages/cli
# Use package manager to install only production dependencies

# Use package manager to install only production dependencies
RUN pnpm --filter "./pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api" deploy /temp-deploy --legacy

# Fix: pnpm deploy --legacy doesn't copy native bindings, so rebuild them from source
# Generic solution: Find and rebuild all packages with native bindings (those with binding.gyp)
RUN echo "=== Rebuilding native modules ===" && \
cd /temp-deploy && \
found_native=0 && \
for dir in $(find node_modules -name "binding.gyp" -type f 2>/dev/null | xargs -r dirname); do \
found_native=1; \
package_name=$(basename $(dirname "$dir")); \
echo "Found native module: $package_name in $dir"; \
if (cd "$dir" && npm rebuild 2>&1); then \
echo "✓ Successfully rebuilt $package_name"; \
else \
echo "⚠ Warning: Failed to rebuild $package_name (may be optional)"; \
fi; \
done && \
if [ $found_native -eq 0 ]; then \
echo "No native modules found (this is normal if your deps don't have native bindings)"; \
fi && \
echo "=== Native modules rebuild complete ==="
RUN cp -r /temp-deploy/node_modules /application/node_modules
RUN chown -R moose:moose /application/node_modules

# No TypeScript path transformations needed
USER root:root
# Clean up temporary directories
RUN rm -rf /temp-deploy /temp-monorepo

RUN if [ -d "/application/node_modules/@514labs/moose-lib/dist/" ]; then ls -la /application/node_modules/@514labs/moose-lib/dist/; fi
USER moose:moose
WORKDIR /application

# https://stackoverflow.com/questions/70096208/dockerfile-copy-folder-if-it-exists-conditional-copy/70096420#70096420
COPY --chown=moose:moose pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api/project.tom[l] ./project.toml
COPY --chown=moose:moose pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api/moose.config.tom[l] ./moose.config.toml
COPY --chown=moose:moose pipeline-registry/dutchie-to-clickhouse/v001/514-labs/typescript/open-api/.moose/packager/versions .moose/versions


# Placeholder for the language specific install command
# Dependencies copied from monorepo build stage


# all commands from here on will be run as the moose user
USER moose:moose

# Checks that the project is valid
RUN which moose
RUN moose check --write-infra-map || (echo "Error running moose check" && exit 1)

# Expose the ports on which the application will listen
EXPOSE 4000

# Set the command to run the application
CMD ["moose", "prod"]
6 changes: 6 additions & 0 deletions connector-registry/apex-trading/_meta/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# apex-trading (Registry)

This is the top-level registry entry for `apex-trading`.

- Versions live under `apex-trading/v1`
- See author implementations under `apex-trading/v1/514-labs`
9 changes: 9 additions & 0 deletions connector-registry/apex-trading/_meta/connector.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"$schema": "https://schemas.connector-factory.dev/connector-root.schema.json",
"identifier": "apex-trading",
"name": "apex-trading",
"category": "api",
"tags": [],
"description": "",
"homepage": ""
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Changelog

All notable changes to this connector will be documented in this file.

## 0.1.0 - Scaffold created

- Initial scaffold for `apex-trading` (TypeScript) added.
18 changes: 18 additions & 0 deletions connector-registry/apex-trading/v1/514-labs/_meta/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
MIT License

Copyright (c) 514-labs

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
10 changes: 10 additions & 0 deletions connector-registry/apex-trading/v1/514-labs/_meta/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# apex-trading Connector (by 514-labs)

This directory contains language-agnostic metadata and documentation for the `apex-trading` connector.

- Name: `apex-trading`
- Author: `514-labs`
- Category: `api`
- Languages: `typescript`

See `_meta/connector.json` for connector metadata and implementation folders for language-specific code.
22 changes: 22 additions & 0 deletions connector-registry/apex-trading/v1/514-labs/_meta/connector.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"name": "apex-trading",
"author": "514-labs",
"version": "v1",
"languages": [
"typescript"
],
"category": "api",
"capabilities": {
"extract": true,
"transform": false,
"load": false
},
"source": {
"type": "api",
"spec": "",
"homepage": ""
},
"tags": [],
"maintainers": [],
"issues": { "typescript": { "default": "" } }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"identifier": "apex-trading",
"name": "apex-trading",
"author": "514-labs",
"version": "v1",
"language": "typescript",
"implementations": ["default"]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Apex Trading API credentials
APEX_API_KEY=your_api_key_here

# Optional: Custom base URL
# APEX_BASE_URL=https://api-docs.apextrading.com

Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
node_modules/
dist/
.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Connector Implementation Guide

This guide walks you through implementing your `apex-trading` connector from scaffold to production.

**📖 For detailed step-by-step instructions, see:**
`connector-registry/_scaffold/CONNECTOR_IMPLEMENTATION_GUIDE.md`

Or view online: [Connector Implementation Guide](../../../../../../_scaffold/CONNECTOR_IMPLEMENTATION_GUIDE.md)

## Quick Checklist

### Phase 1: Understand Your API ✓
- [ ] Read the `apex-trading` API documentation
- [ ] Identify authentication method
- [ ] Understand pagination pattern
- [ ] Note rate limits
- [ ] Identify key endpoints

### Phase 2: Scaffold ✓
- [x] Connector scaffolded

### Phase 3: Configure Authentication
- [ ] Update `ConnectorConfig` type in `src/client/connector.ts`
- [ ] Implement `init()` method to set up auth
- [ ] Update `.env.example` with required credentials

### Phase 4: Implement Pagination
- [ ] Choose pagination pattern (offset/cursor/page)
- [ ] Implement in `src/lib/paginate.ts`
- [ ] Test with API to verify it works

### Phase 5: Implement Resources
- [ ] Define resource types in `src/resources/products.ts`
- [ ] Implement list/get/create methods as needed
- [ ] Add resource getters to connector class

### Phase 6: Add Schemas
- [ ] Create JSON schemas in `schemas/raw/json/`
- [ ] Add documentation `.md` files
- [ ] Register in `schemas/index.json`

### Phase 7: Update Documentation
- [ ] Update `README.md` with examples
- [ ] Document config in `docs/configuration.md`
- [ ] Update `.env.example`

### Phase 8: Write Tests
- [ ] Update `tests/resource.test.ts`
- [ ] Add observability tests if using logging/metrics
- [ ] Test pagination edge cases

### Phase 9: Build & Test
- [ ] Run `pnpm run build` - should complete without errors
- [ ] Run `pnpm test` - all tests should pass
- [ ] Try `examples/basic-usage.ts` with real API

### Phase 10: Real-World Testing
- [ ] Test with real credentials
- [ ] Verify pagination with large datasets
- [ ] Test error handling
- [ ] Test rate limiting (if applicable)

## Common Patterns

See the detailed guide for code examples of:
- Simple API key authentication
- Bearer token authentication
- OAuth2 authentication
- Offset-based pagination
- Cursor-based pagination
- Page number pagination

## Reference Connectors

Look at these for examples:
- **Socrata** (`connector-registry/socrata/`): Simple API with offset pagination
- **Meta Ads** (`connector-registry/meta-ads/`): OAuth2 with cursor pagination
- **Google Analytics** (`connector-registry/google-analytics/`): Service account auth

## Need Help?

Refer to the full implementation guide at:
`connector-registry/_scaffold/CONNECTOR_IMPLEMENTATION_GUIDE.md`
Loading