From 80b7aed89abd93c464d88bf21b7a5af16f5f7f91 Mon Sep 17 00:00:00 2001 From: eren-karakus0 Date: Thu, 19 Feb 2026 12:00:33 +0300 Subject: [PATCH] feat: add batch upload example for uploading multiple files Add a new example app that demonstrates how to upload multiple files to Shelby in a single run. The script scans a directory and uploads all files sequentially with progress output and per-file error handling. Features: - Automatic file discovery from assets directory - Per-file error handling (one failure doesn't stop the rest) - Duplicate blob detection (skips already-uploaded files) - File size display and upload summary --- apps/batch-upload/.env.example | 2 + apps/batch-upload/README.md | 100 +++++++++++++++++++++++++++ apps/batch-upload/assets/hello.txt | 1 + apps/batch-upload/assets/sample.json | 4 ++ apps/batch-upload/package.json | 34 +++++++++ apps/batch-upload/src/index.ts | 85 +++++++++++++++++++++++ apps/batch-upload/tsconfig.json | 25 +++++++ 7 files changed, 251 insertions(+) create mode 100644 apps/batch-upload/.env.example create mode 100644 apps/batch-upload/README.md create mode 100644 apps/batch-upload/assets/hello.txt create mode 100644 apps/batch-upload/assets/sample.json create mode 100644 apps/batch-upload/package.json create mode 100644 apps/batch-upload/src/index.ts create mode 100644 apps/batch-upload/tsconfig.json diff --git a/apps/batch-upload/.env.example b/apps/batch-upload/.env.example new file mode 100644 index 0000000..453ad6e --- /dev/null +++ b/apps/batch-upload/.env.example @@ -0,0 +1,2 @@ +SHELBY_ACCOUNT_PRIVATE_KEY=ed25519-priv-0xYourPrivateKeyHere +SHELBY_API_KEY=AG-YourAPIKeyHere diff --git a/apps/batch-upload/README.md b/apps/batch-upload/README.md new file mode 100644 index 0000000..856ad78 --- /dev/null +++ b/apps/batch-upload/README.md @@ -0,0 +1,100 @@ +# Shelby Batch Upload Example + +An example application demonstrating how to upload multiple files to Shelby in a single run. Place your files in the `assets/` directory and the script will upload them all sequentially. + +## Prerequisites + +- Node.js >= 22 +- pnpm package manager +- A Shelby account with sufficient balance for blob storage +- Shelby API key +- Shelby account private key + +## Installation + +1. Clone the repository and navigate to the batch-upload directory: + ```bash + cd apps/batch-upload + ``` + +2. Install dependencies: + ```bash + pnpm install + ``` + +## Environment Variables + +Create a `.env` file in the root of this project directory with the following required environment variables. You can copy the `.env.example` file as a starting point: + +```bash +cp .env.example .env +``` + +Then update the values in your `.env` file: + +```env +SHELBY_ACCOUNT_PRIVATE_KEY=your_private_key_here +SHELBY_API_KEY=your_api_key_here +``` + +More information on obtaining an API key on the [Shelby docs site](https://docs.shelby.xyz/sdks/typescript/acquire-api-keys). + +## Configuration + +You can modify the behavior by changing the configuration constants in `src/index.ts`: + +```typescript +// Directory containing files to upload. +const UPLOAD_DIR = join(process.cwd(), "assets"); +// How long before each upload expires (in microseconds from now). +const TIME_TO_LIVE = 60 * 60 * 1_000_000; // 1 hour +// Optional prefix for blob names on Shelby. +const BLOB_PREFIX = ""; +``` + +## Usage + +1. Place files you want to upload in the `assets/` directory (sample files are included). + +2. Run the batch upload: + ```bash + pnpm upload + ``` + +### Example Output + +``` +Found 2 file(s) to upload + + Uploading hello.txt (41 B)... done + Uploading sample.json (89 B)... done + +āœ“ 2 uploaded, 0 failed +``` + +## How It Works + +1. **Environment Validation**: Validates that all required environment variables are set +2. **Client Initialization**: Creates a Shelby client instance connected to the Shelbynet network +3. **File Discovery**: Scans the `assets/` directory for all files (hidden files are excluded) +4. **Sequential Upload**: Uploads each file to Shelby with the configured expiration time +5. **Error Handling**: Gracefully handles errors per file ā€” if one upload fails, the rest continue +6. **Duplicate Detection**: Skips files that already exist on Shelby instead of failing + +## Troubleshooting + +### Common Issues + +1. **No files found** + - Ensure your files are placed directly in the `assets/` directory + - Hidden files (starting with `.`) are excluded by default + +2. **Blob already exists** + - The script automatically skips existing blobs and counts them as successful + +3. **Insufficient balance** + - Fund your account using the [ShelbyUSD faucet](https://docs.shelby.xyz/apis/faucet/shelbyusd) and [APT faucet](https://docs.shelby.xyz/apis/faucet/aptos) + +4. **Rate limit exceeded (429)** + - Wait a moment before retrying + - Consider obtaining your own API key for higher limits diff --git a/apps/batch-upload/assets/hello.txt b/apps/batch-upload/assets/hello.txt new file mode 100644 index 0000000..f922105 --- /dev/null +++ b/apps/batch-upload/assets/hello.txt @@ -0,0 +1 @@ +Hello from Shelby batch upload example\! diff --git a/apps/batch-upload/assets/sample.json b/apps/batch-upload/assets/sample.json new file mode 100644 index 0000000..b6941f9 --- /dev/null +++ b/apps/batch-upload/assets/sample.json @@ -0,0 +1,4 @@ +{ + "message": "sample JSON data for Shelby storage", + "timestamp": "2026-01-01T00:00:00Z" +} diff --git a/apps/batch-upload/package.json b/apps/batch-upload/package.json new file mode 100644 index 0000000..37d9d39 --- /dev/null +++ b/apps/batch-upload/package.json @@ -0,0 +1,34 @@ +{ + "name": "@shelby-protocol/batch-upload-example", + "version": "1.0.0", + "description": "An example app to demonstrate uploading multiple files to Shelby in a single run", + "type": "module", + "scripts": { + "upload": "tsx --env-file=.env src/index.ts", + "lint": "biome check .", + "fmt": "biome check . --write" + }, + "keywords": [ + "shelby", + "sdk", + "blob", + "storage", + "batch", + "upload" + ], + "author": "eren-karakus0", + "license": "MIT", + "dependencies": { + "@aptos-labs/ts-sdk": "^5.1.1", + "@shelby-protocol/sdk": "latest" + }, + "devDependencies": { + "@biomejs/biome": "2.2.4", + "tsx": "^4.20.5", + "typescript": "^5.9.2", + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22" + } +} diff --git a/apps/batch-upload/src/index.ts b/apps/batch-upload/src/index.ts new file mode 100644 index 0000000..792cbd2 --- /dev/null +++ b/apps/batch-upload/src/index.ts @@ -0,0 +1,85 @@ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { Account, Ed25519PrivateKey, Network } from "@aptos-labs/ts-sdk"; +import { ShelbyNodeClient } from "@shelby-protocol/sdk/node"; + +// Directory containing files to upload. +const UPLOAD_DIR = join(process.cwd(), "assets"); +// How long before each upload expires (in microseconds from now). +const TIME_TO_LIVE = 60 * 60 * 1_000_000; // 1 hour +// Optional prefix for blob names on Shelby. +const BLOB_PREFIX = ""; + +if (!process.env.SHELBY_ACCOUNT_PRIVATE_KEY) { + throw new Error("Missing SHELBY_ACCOUNT_PRIVATE_KEY"); +} +if (!process.env.SHELBY_API_KEY) { + throw new Error("Missing SHELBY_API_KEY"); +} + +// 1) Initialize a Shelby client (auth via API key; target shelbynet). +const client = new ShelbyNodeClient({ + network: Network.SHELBYNET, + apiKey: process.env.SHELBY_API_KEY, +}); + +// 2) Create an Aptos account object from your private key. +const signer = Account.fromPrivateKey({ + privateKey: new Ed25519PrivateKey(process.env.SHELBY_ACCOUNT_PRIVATE_KEY), +}); + +// 3) Discover all files in the assets directory. +const files = readdirSync(UPLOAD_DIR) + .filter((name) => !name.startsWith(".")) + .filter((name) => statSync(join(UPLOAD_DIR, name)).isFile()); + +if (files.length === 0) { + console.log("No files found in", UPLOAD_DIR); + process.exit(0); +} + +console.log(`Found ${files.length} file(s) to upload\n`); + +// 4) Upload each file sequentially. +let uploaded = 0; +let failed = 0; + +for (const file of files) { + const filePath = join(UPLOAD_DIR, file); + const blobName = `${BLOB_PREFIX}${file}`; + const size = statSync(filePath).size; + + try { + process.stdout.write(` Uploading ${blobName} (${formatBytes(size)})...`); + + await client.upload({ + blobData: readFileSync(filePath), + signer, + blobName, + expirationMicros: Date.now() * 1000 + TIME_TO_LIVE, + }); + + console.log(" done"); + uploaded++; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + + if (msg.includes("EBLOB_WRITE_CHUNKSET_ALREADY_EXISTS")) { + console.log(" skipped (already exists)"); + uploaded++; + continue; + } + + console.log(" FAILED"); + console.error(` Error: ${msg}`); + failed++; + } +} + +console.log(`\nāœ“ ${uploaded} uploaded, ${failed} failed`); + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/apps/batch-upload/tsconfig.json b/apps/batch-upload/tsconfig.json new file mode 100644 index 0000000..dac2510 --- /dev/null +++ b/apps/batch-upload/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "allowJs": true, + "strict": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": false, + "lib": ["ES2022", "DOM"], + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts"] +}