From db2dfb5f2c1044ccf4d0f1585dbd4a165e8e583e Mon Sep 17 00:00:00 2001 From: Isaac Onyemaechi Date: Tue, 2 Dec 2025 04:13:03 +0100 Subject: [PATCH] feat: add KYC data migration script from CSV to Supabase --- scripts/MIGRATION_README.md | 117 ++++++++++++++++++++++++ scripts/migrate-kyc-data.ts | 172 ++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100644 scripts/MIGRATION_README.md create mode 100644 scripts/migrate-kyc-data.ts diff --git a/scripts/MIGRATION_README.md b/scripts/MIGRATION_README.md new file mode 100644 index 00000000..baac6a61 --- /dev/null +++ b/scripts/MIGRATION_README.md @@ -0,0 +1,117 @@ +# KYC Data Migration from CSV + +This script migrates user KYC data from a local CSV file into the Supabase `user_kyc_profiles` table. + +## Prerequisites + +### 1. Install Dependencies + +If you haven't already, install the required packages: +```bash +npm install @supabase/supabase-js dotenv ts-node typescript @types/node csv-parse +``` + +### 2. Environment Variables + +Ensure your `.env` file contains the following Supabase credentials: + +```bash +# Supabase +NEXT_PUBLIC_SUPABASE_URL=your_supabase_url +SUPABASE_SERVICE_ROLE_KEY=your_service_role_key +``` + +### 3. CSV Data File + +Place a CSV file named `kyc-data.csv` inside the `scripts/` directory. The script expects the file to have a header row with column names that match the `CsvRow` interface in the script. + +**Expected CSV Columns:** +- `user_id` (maps to `wallet_address`) +- `id_type` +- `country` + +## Usage + +### Dry Run (Recommended First) + +To preview the data that will be inserted without writing anything to the database, run: + +```bash +npx ts-node scripts/migrate-kyc-data.ts --dry-run +``` + +This command will: +- Read and parse `kyc-data.csv`. +- Transform the first 5 records into the database format. +- Print the transformed data to the console. +- **It will NOT write any data to Supabase.** + +### Full Migration + +Once you have verified that the dry run output is correct, you can perform the full migration: + +```bash +npx ts-node scripts/migrate-kyc-data.ts +``` + +This command will: +1. Read all records from `kyc-data.csv`. +2. Transform each record to match the `user_kyc_profiles` schema. +3. Upsert each record into the Supabase table, using `wallet_address` to handle conflicts. + +## How It Works + +### 1. Extraction +- The script reads the `kyc-data.csv` file from the `scripts/` directory. +- It uses the `csv-parse` library to parse the file into an array of JavaScript objects. + +### 2. Transformation +For each row in the CSV: +- It maps the CSV columns to the fields in the `user_kyc_profiles` table. +- `user_id` from the CSV is used as the `wallet_address`. +- It defaults to `tier: 2` for all migrated users. + +### 3. Loading +- The script connects to your Supabase instance using the service role key. +- It iterates through the transformed records and uses `supabase.from('user_kyc_profiles').upsert(...)` to insert or update each one. +- The `onConflict: 'wallet_address'` option ensures that existing records are updated, preventing duplicates. + +## Data Mapping + +| CSV Column | New Schema Field | Notes | +|----------------|--------------------|--------------------------------------------| +| `user_id` | `wallet_address` | Primary key, lowercased for consistency. | +| `id_type` | `id_type` | | +| `country` | `id_country` | | +| `smile_job_id` | `smile_job_id` | | +| `verified_at` | `verified_at` | Also sets `verified` to `true`. | +| - | `tier` | Hardcoded to `2` for all records. | + +## Troubleshooting + +### `CSV file not found` +**Issue**: The script throws an error `CSV file not found at: `. +**Solution**: Make sure your CSV file is named exactly `kyc-data.csv` and is located in the `/Users/prof/Documents/paycrest/noblocks/scripts/` directory. + +### `Missing Supabase credentials` +**Issue**: The script throws an error about missing credentials. +**Solution**: Ensure your `.env` file is in the root of the `noblocks` project and contains the correct `NEXT_PUBLIC_SUPABASE_URL` and `SUPABASE_SERVICE_ROLE_KEY`. + +### Data not appearing as expected +**Issue**: The data in Supabase doesn't look right. +**Solution**: +1. Run the script with `--dry-run` and inspect the JSON output in your console. +2. Check that the column headers in your `kyc-data.csv` file exactly match the expected names (e.g., `user_id`, `phone_number`). +3. Verify the data formats in the CSV (e.g., dates in `verified_at` are valid). + +## Rollback + +If you need to undo the migration, you can run a SQL query in the Supabase SQL Editor. Be very careful with this operation. + +```sql +-- Example: Delete records that were created or updated by the script. +-- You might need to adjust the timestamp to match your migration time. +DELETE FROM public.user_kyc_profiles WHERE updated_at >= '2025-12-02 00:00:00+00'; +``` + +It is safer to identify the migrated records by a set of `wallet_address` values if possible. diff --git a/scripts/migrate-kyc-data.ts b/scripts/migrate-kyc-data.ts new file mode 100644 index 00000000..9befb34f --- /dev/null +++ b/scripts/migrate-kyc-data.ts @@ -0,0 +1,172 @@ +/** + * Migrate minimal KYC fields from a CSV export to Supabase + * - Reads CSV with headers: Job ID, User ID, Country, ID Type, Result + * - Filters rows where Result === "Approved" + * - Upserts into public.user_kyc_profiles: + * - wallet_address ← User ID (lowercased) + * - id_country ← Country + * - id_type ← ID Type + * - verified=true, verified_at=now (optional; remove if not desired) + * + * Usage: + * npx ts-node scripts/migrate-kyc-data.ts --dry-run + * npx ts-node scripts/migrate-kyc-data.ts --csv ./kyc-export.csv + */ + +import { createClient } from '@supabase/supabase-js'; +import * as dotenv from 'dotenv'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { parse } from 'csv-parse/sync'; + +dotenv.config(); + +// ESM-safe path resolution +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Env +const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL; +const SUPABASE_SERVICE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY; + +if (!SUPABASE_URL || !SUPABASE_SERVICE_KEY) { + throw new Error('Missing Supabase credentials in .env (NEXT_PUBLIC_SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY)'); +} + +const supabase = createClient(SUPABASE_URL, SUPABASE_SERVICE_KEY); + +// CLI: allow overriding the CSV path via --csv +const csvArgIndex = process.argv.findIndex((a) => a === '--csv'); +const CSV_FILE_PATH = csvArgIndex >= 0 + ? path.resolve(process.argv[csvArgIndex + 1]) + : path.join(__dirname, 'kyc-export.csv'); // default name next to script + + +interface CsvRowRaw { + 'Job ID'?: string; + 'User ID'?: string; + 'SDK'?: string; + 'Date'?: string; + 'Timestamp'?: string; + 'Job Time'?: string; + 'Product'?: string; + 'Job Type'?: string; + 'Country'?: string; + 'ID Type'?: string; + 'Result'?: string; + 'Message'?: string; + 'SmartCheck User'?: string; +} + +type CsvRow = { + job_id: string; + user_id: string; + country?: string | null; + id_type?: string | null; + result: string; +}; + +// Read + normalize CSV +function readCsv(): CsvRow[] { + if (!fs.existsSync(CSV_FILE_PATH)) { + throw new Error(`CSV file not found at: ${CSV_FILE_PATH}`); + } + console.log(` Reading data from ${CSV_FILE_PATH}`); + + const content = fs.readFileSync(CSV_FILE_PATH); + const raw = parse(content, { + columns: true, + skip_empty_lines: true, + trim: true, + }) as CsvRowRaw[]; + + console.log(`Found ${raw.length} records in CSV file.`); + + const rows: CsvRow[] = raw.map((r) => ({ + job_id: (r['Job ID'] || '').trim(), + user_id: (r['User ID'] || '').trim(), + country: r['Country'] ? r['Country'].trim() : null, + id_type: r['ID Type'] ? r['ID Type'].trim() : null, + result: (r['Result'] || '').trim(), + })); + + const valid = rows.filter((r) => r.job_id && r.user_id && r.result); + const skipped = rows.length - valid.length; + if (skipped > 0) { + console.warn(` Skipped ${skipped} rows missing Job ID, User ID, or Result`); + } + return valid; +} + + +function buildRow(r: CsvRow) { + const isApproved = r.result === 'Approved'; + const nowISO = new Date().toISOString(); + + return { + wallet_address: r.user_id.toLowerCase(), + id_country: r.country || null, + id_type: r.id_type || null, + platform: [ + { + type: 'id', + identifier: 'smile_id', + reference: '', + } + ], + verified: isApproved, + verified_at: isApproved ? nowISO : null, + updated_at: nowISO, + }; +} + +// Upsert — conflict target: wallet_address (PK) +async function upsertRows(rows: any[]) { + console.log(`\nUpserting ${rows.length} records into public.user_kyc_profiles...`); + let ok = 0, failed = 0; + + for (const row of rows) { + const { error } = await supabase + .from('user_kyc_profiles') + .upsert(row, { onConflict: 'wallet_address' }); + + if (error) { + console.error(`❌ ${row.wallet_address}: ${error.message}`); + failed++; + } else { + console.log(`✅ ${row.wallet_address}`); + ok++; + } + } + console.log(`\n Summary: OK=${ok}, Failed=${failed}`); +} + +async function main() { + const isDryRun = process.argv.includes('--dry-run'); + console.log(isDryRun ? 'Running in DRY RUN mode.' : 'Running in LIVE mode.'); + + try { + const all = readCsv(); + const approved = all.filter((r) => r.result === 'Approved'); + console.log(`✅ Approved rows: ${approved.length}`); + + const rows = approved.map(buildRow); + + if (isDryRun) { + console.log('--- Dry Run Output (first 5 rows) ---'); + console.log(JSON.stringify(rows.slice(0, 5), null, 2)); + console.log('-------------------------------------'); + console.log('No data will be written to the database in dry run mode.'); + } else { + await upsertRows(rows); + } + + console.log('🎉 Migration script finished.'); + } catch (err: any) { + console.error(' An error occurred:', err.message || err); + process.exit(1); + } +} + +main(); \ No newline at end of file