diff --git a/.github/workflows/migrate-prod.yml b/.github/workflows/migrate-prod.yml index e812f87..bad7765 100644 --- a/.github/workflows/migrate-prod.yml +++ b/.github/workflows/migrate-prod.yml @@ -20,6 +20,11 @@ on: required: true type: string default: 'farajaland.opencrvs.org' + skip: + description: 'Number of records to skip' + required: false + type: number + default: 0 jobs: migrate: @@ -53,6 +58,7 @@ jobs: env: OPENCRVS_EVENT: ${{ github.event.inputs.event }} OPENCRVS_DOMAIN: ${{ github.event.inputs.domain }} + RECORD_SKIP: ${{ github.event.inputs.skip }} OPENCRVS_CLIENT_ID: ${{ secrets.OPENCRVS_CLIENT_ID }} OPENCRVS_CLIENT_SECRET: ${{ secrets.OPENCRVS_CLIENT_SECRET }} run: | diff --git a/v1-to-v2-data-migration/helpers/transform.ts b/v1-to-v2-data-migration/helpers/transform.ts index 67a512b..03d6fdd 100644 --- a/v1-to-v2-data-migration/helpers/transform.ts +++ b/v1-to-v2-data-migration/helpers/transform.ts @@ -526,8 +526,21 @@ function postProcess( const approvedCorrections = [] const rev = document.actions.slice().reverse() + let firstRegisterFound = false for (const action of rev) { + if (action.type === 'REGISTER' && action.registrationNumber) { + if (firstRegisterFound) { + console.warn( + `Multiple REGISTER actions found for document ${document.id}` + ) + action.registrationNumber = undefined + action.status = 'Requested' + } else { + firstRegisterFound = true + } + } + if (action.type === 'APPROVE_CORRECTION') { approvedCorrections.push(action.requestId) } diff --git a/v1-to-v2-data-migration/helpers/vars.ts b/v1-to-v2-data-migration/helpers/vars.ts index bf84e20..28257ed 100644 --- a/v1-to-v2-data-migration/helpers/vars.ts +++ b/v1-to-v2-data-migration/helpers/vars.ts @@ -1,6 +1,7 @@ // @ts-nocheck - Using Deno-specific environment variables export const DOMAIN = Deno?.env?.get('OPENCRVS_DOMAIN') || 'localhost' export const EVENT = Deno?.env?.get('OPENCRVS_EVENT') || 'birth' +export const RECORD_SKIP = Number(Deno?.env?.get('RECORD_SKIP')) || 0 export const CLIENT_ID = Deno?.env?.get('OPENCRVS_CLIENT_ID') export const CLIENT_SECRET = Deno?.env?.get('OPENCRVS_CLIENT_SECRET') export const ADMIN_USERNAME = diff --git a/v1-to-v2-data-migration/migrate.ipynb b/v1-to-v2-data-migration/migrate.ipynb index 80598e8..cc2cf75 100644 --- a/v1-to-v2-data-migration/migrate.ipynb +++ b/v1-to-v2-data-migration/migrate.ipynb @@ -192,8 +192,7 @@ "} from './helpers/gqlHandlers.ts'\n", "import { transform } from './helpers/transform.ts'\n", "import { batch, getIndexErrors } from './helpers/utils.ts'\n", - "\n", - "const saved = []\n", + "import { RECORD_SKIP } from './helpers/vars.ts'\n", "\n", "const migrateBirth = async (entryIds) => {\n", " const transformed = []\n", @@ -231,12 +230,16 @@ "}\n", "\n", "if (EVENT === 'birth') {\n", + " const skippedPages = RECORD_SKIP ? Math.floor(RECORD_SKIP / 1000) : 0\n", " const pageSize = 1000\n", " const batchSize = 100\n", " let itemsRemaining = 0\n", - " let page = 1\n", - " let totalProcessed = 0\n", + " let page = 1 + skippedPages\n", + " let totalProcessed = skippedPages * pageSize\n", "\n", + " if (RECORD_SKIP) {\n", + " console.log(`Skipping first ${skippedPages * pageSize} records`)\n", + " }\n", " do {\n", " const birthRegistrations = await fetchAllBirthRegistrations(\n", " sysToken,\n", @@ -250,6 +253,7 @@ "\n", " const { results, totalItems } = birthRegistrations.data.searchEvents\n", " const birthIds = results.map((x) => x.id)\n", + "\n", " console.log(\n", " `Processing next page of ${birthIds.length} of ${totalItems} total records`\n", " )\n", @@ -340,11 +344,16 @@ "}\n", "\n", "if (EVENT === 'death') {\n", + " const skippedPages = RECORD_SKIP ? Math.floor(RECORD_SKIP / 1000) : 0\n", " const pageSize = 1000\n", " const batchSize = 100\n", " let itemsRemaining = 0\n", - " let page = 1\n", - " let totalProcessed = 0\n", + " let page = 1 + skippedPages\n", + " let totalProcessed = skippedPages * pageSize\n", + "\n", + " if (RECORD_SKIP) {\n", + " console.log(`Skipping first ${skippedPages * pageSize} records`)\n", + " }\n", "\n", " do {\n", " const deathRegistrations = await fetchAllDeathRegistrations(\n",