From fb4da036dbd93dce0b8f3ab24b39119f4e291ded Mon Sep 17 00:00:00 2001 From: Vadym Mudryi Date: Wed, 28 Jan 2026 15:56:03 +0200 Subject: [PATCH] feat: Migration from swarm to k8s --- .../secret-mapping-opencrvs-deps.yml | 7 +- .github/workflows/deploy-opencrvs.yml | 8 + infrastructure/environments/swarm-to-k8s.ts | 72 +++++++++ infrastructure/environments/templates.ts | 137 +++++++++--------- .../environments/update-workflows.ts | 33 ++--- .../tasks/k8s/install-containerd.yml | 6 + package.json | 4 +- yarn.lock | 37 +++++ 8 files changed, 209 insertions(+), 95 deletions(-) create mode 100644 infrastructure/environments/swarm-to-k8s.ts diff --git a/.github/TEMPLATES/secret-mapping-opencrvs-deps.yml b/.github/TEMPLATES/secret-mapping-opencrvs-deps.yml index 16abd61c..ae4343cb 100644 --- a/.github/TEMPLATES/secret-mapping-opencrvs-deps.yml +++ b/.github/TEMPLATES/secret-mapping-opencrvs-deps.yml @@ -33,12 +33,15 @@ kibana-users-secret: - KIBANA_USERNAME - KIBANA_PASSWORD +# Traefik static SSL certificate +# backward compatible with existing implementation, +# See: https://documentation.opencrvs.org/v1.8/setup/3.-installation/3.3-set-up-a-server-hosted-environment/3.3.5-setup-dns-a-records/4.3.2.3-static-tls-certificates traefik-cert: type: tls namespace: traefik data: - - TRAEFIK_CERT: cert - - TRAEFIK_KEY: key + - SSL_CRT: cert + - SSL_KEY: key # If backup is configured then workflow will use GitHub secrets for current environment # If restore is configured then workflow will fetch secrets from source environment (usually production) diff --git a/.github/workflows/deploy-opencrvs.yml b/.github/workflows/deploy-opencrvs.yml index 6cd9a7d6..52c2c737 100644 --- a/.github/workflows/deploy-opencrvs.yml +++ b/.github/workflows/deploy-opencrvs.yml @@ -7,6 +7,8 @@ on: type: string countryconfig-image-tag: type: string + data-seed-enabled: + type: boolean environment: type: string workflow_dispatch: @@ -19,6 +21,11 @@ on: description: "Tag of the countryconfig image" required: true default: "v1.9.6" + data-seed-enabled: + description: "Enable data seeding during deployment" + required: false + default: "true" + type: boolean environment: description: "Target environment" required: true @@ -140,6 +147,7 @@ jobs: --set countryconfig.image.tag="$COUNTRYCONFIG_IMAGE_TAG" \ --set countryconfig.image.name="$COUNTRYCONFIG_IMAGE_NAME" \ --set data_seed.env.ACTIVATE_USERS="${{ vars.ACTIVATE_USERS || 'false' }}" \ + --set data_seed.enabled="${{ inputs.data-seed-enabled }}" \ --set hostname=${{ vars.DOMAIN }} 2>&1 ; STATUS=$?; kill $STERN_PID 2>/dev/null || true exit $STATUS diff --git a/infrastructure/environments/swarm-to-k8s.ts b/infrastructure/environments/swarm-to-k8s.ts new file mode 100644 index 00000000..833ab7da --- /dev/null +++ b/infrastructure/environments/swarm-to-k8s.ts @@ -0,0 +1,72 @@ +import * as path from 'path'; +import kleur from 'kleur' +import { error, info, log, success, warn } from './logger' +import { updateWorkflowEnvironments } from './update-workflows'; +import { generateInventory, copyChartsValues, extractAndModifyUsers, extractWorkerNodes, extractBackupNode, dockerManagerFirst, readYamlFile } from './templates' + + + +(async () => { + log(kleur.bold( + "------------------------------------------------\n" + + "OpenCRVS Infrastructure migration script: \n" + + "Migrating Swarm configurations to Kubernetes\n" + + "------------------------------------------------\n" + )); + const environment_type = process.env.ENVIRONMENT_TYPE || 'production'; + const environment = process.env.ENVIRONMENT || ''; + if (!environment) { + error('\n', 'Environment variable ENVIRONMENT is not set. Exiting.'); + process.exit(1); + } + if (["backup", "jumpbox"].includes(environment)) { + info(` > ${environment} environment will not be migrated, see migration notes`) + process.exit(0); + } + log(kleur.bold().underline('Migration properties:')); + log(` ✓ Environment: ${environment}`) + + const old_inventory_path = process.env.OLD_INVENTORY_PATH || ''; + if (!old_inventory_path) { + error('\n', 'Environment variable OLD_INVENTORY_PATH is not set. Exiting.'); + log('\n', 'Old inventory path is required to read existing Swarm configurations.'); + process.exit(1); + } + const ansible_inventory = path.join(old_inventory_path, environment + '.yml'); + const data = readYamlFile(ansible_inventory) as any; + log(` ✓ Loaded old inventory file: ${ansible_inventory}`); + + const master = dockerManagerFirst(data) || '' + log(` ✓ Kubernetes API Host (Docker Manager): ${master}`); + const users = extractAndModifyUsers(data); + const worker_nodes = extractWorkerNodes(data); + log(` ✓ Worker nodes: ${worker_nodes.join(', ')}`); + const backup_host = extractBackupNode(data); + log(` ✓ Backup host: ${backup_host}`); + + generateInventory( + environment, + { + worker_nodes: worker_nodes, + users: users, + backup_host: backup_host, + kube_api_host: master + } + ) + + copyChartsValues( + environment, + { + env: environment, + environment_type: environment_type, + // FIXME: In general that should be environment_type, + // Hardcode like this blocks us from being generic: + // https://github.com/opencrvs/opencrvs-core/issues/11171 + is_qa_env: environment !== 'production' ? "true" : "false", + backup_enabled: environment === 'production' ? "true" : "false", + restore_enabled: environment === 'staging' ? "true" : "false", + restore_environment_name: environment === 'staging' ? "production" : "" + } + ) + await updateWorkflowEnvironments(); +})(); diff --git a/infrastructure/environments/templates.ts b/infrastructure/environments/templates.ts index 9d6fbab1..6b9b2f28 100644 --- a/infrastructure/environments/templates.ts +++ b/infrastructure/environments/templates.ts @@ -1,18 +1,59 @@ import fs from "fs"; import path from "path"; -import { log } from './logger' -/** - * Replace placeholders in file content. - * Customize the replacements map to your needs. - */ -function replacePlaceholders(content: string, replacements: Record): string { - let updated = content; - for (const [key, value] of Object.entries(replacements)) { - const regex = new RegExp(`\\{\\{${key}\\}\\}`, "g"); // matches ${KEY} - let clear_value = String(value).replace(/[\x00-\x1F\x7F]/g, ""); // remove control characters - updated = updated.replace(regex, clear_value); +import { log, success, warn } from './logger' +import * as yaml from 'js-yaml'; +import Handlebars from 'handlebars'; + +// Register a helper to increment numbers +Handlebars.registerHelper('data_label_idx', function(value) { + return parseInt(value) + 2; +}); + +export function readYamlFile(filePath: any): any { + const fileContent = fs.readFileSync(filePath, "utf8"); + return yaml.load(fileContent); +} + + +// Extract users from the old inventory +export function extractAndModifyUsers(data: any): any { + if (!data?.all?.vars?.users) { + return { users: [] }; + } + return data.all.vars.users; +} + +export function dockerManagerFirst(data: any): string { + if (!data?.['docker-manager-first']?.hosts) { + throw new Error('Invalid YAML structure: missing docker-manager-first.hosts'); + } + const hosts = data['docker-manager-first'].hosts; + const dockerManagerFirst = Object.values(hosts) + .filter((host: any) => host.ansible_host) + .map((host: any) => host.ansible_host); + return dockerManagerFirst.length === 1 ? dockerManagerFirst[0] : ''; +} + +export function extractBackupNode(data: any): string { + if (!data?.['backups']?.hosts) { + return ''; + } + const hosts = data['backups'].hosts; + const backupHostEntry = Object.values(hosts) + .filter((host: any) => host.ansible_host) + .map((host: any) => host.ansible_host); + return backupHostEntry.length === 1 ? backupHostEntry[0] : ''; +} + +export function extractWorkerNodes(data: any): string[] { + if (!data?.['docker-workers']?.hosts) { + return []; } - return updated; + const hosts = data['docker-workers'].hosts; + const worker_hosts = Object.values(hosts) + .filter((host: any) => host.ansible_host) + .map((host: any) => host.ansible_host); + return worker_hosts; } /** @@ -23,7 +64,7 @@ function replacePlaceholders(content: string, replacements: Record) { +export function copyChartsValues(env: string, values: Record) { const srcDir = path.resolve(__dirname, "templates", "charts-values"); const destDir = path.resolve(__dirname, "..", "..", "environments", env); fs.mkdirSync(destDir, { recursive: true }); @@ -38,22 +79,23 @@ export function copyChartsValues(env: string, replacements: Record){ // Check if output file already exists if (fs.existsSync(outputPath)) { - log(`⚠️ Skipping ${templatePath}, file already exists at ${outputPath}`); + warn(` ⚠️ Skipping ${templatePath}, file already exists at ${outputPath}`); return; } - let template = fs.readFileSync(templatePath, "utf-8"); + const templateFile = fs.readFileSync(templatePath, "utf-8"); + const template = Handlebars.compile(templateFile); + values['single_node'] = (values['worker_nodes'].length > 0 || values['backup_host']) ? "false" : "true"; - // Extract worker nodes and backup host from values - let worker_nodes = values['worker_nodes'].map((e: string) => String(e) - .replace(/[\x00-\x1F\x7F]/g, "")) - .filter((e: string) => e.length > 0); - - // Generate workers block - if (worker_nodes && worker_nodes.length > 0) { - let workersBlock = ` - # Workers section is optional, for single node cluster feel free to remove this section - # section can be added later - # more workers can be added later as well - workers: - hosts:`; - - worker_nodes.forEach((host: string, index: number) => { - const isFirstWorker = index === 0; - workersBlock += ` - worker${index}: - ansible_host: ${host}${isFirstWorker ? ` - labels: - # By default all datastores are deployed to worker node with role data1 - role: data1` : ''} -`; - }); - - template = template.replace('{{WORKERS_BLOCK}}', workersBlock); - } else { - // No worker nodes, remove the placeholder - template = template.replace('{{WORKERS_BLOCK}}', ''); - } - - - // Generate backup block if backup_host is provided - const backupHost = String(values['backup_host']).replace(/[\x00-\x1F\x7F]/g, ""); - let backupBlock = ''; - if (backupHost.length > 0) { - backupBlock = ` - # backup section is optional, feel free to remove if backups are not enabled - # section can be added later - backup: - hosts: - backup1: - ansible_host: ${backupHost} -`; - } - template = template.replace('{{BACKUP_BLOCK}}', backupBlock); + const updated = template(values); - // Determine if single-node or multi-node - values['single_node'] = (worker_nodes.length > 0 || backupHost) ? "false" : "true"; - const updated = replacePlaceholders(template, values); - values fs.mkdirSync(path.dirname(outputPath), { recursive: true }); fs.writeFileSync(outputPath, updated); - log(`✅ Generated inventory file at ${outputPath}`); + log(`\n✅ Generated inventory file at ${outputPath}\n`); } diff --git a/infrastructure/environments/update-workflows.ts b/infrastructure/environments/update-workflows.ts index 1c997255..b9942bc1 100644 --- a/infrastructure/environments/update-workflows.ts +++ b/infrastructure/environments/update-workflows.ts @@ -3,7 +3,7 @@ import { readFileSync, writeFileSync, statSync, existsSync } from 'fs'; import { basename, join } from 'path'; import * as glob from 'glob'; import * as yaml from 'js-yaml'; - +import { error, info, log, success, warn } from './logger' interface WorkflowConfig { workflows: string[]; path: string; @@ -18,9 +18,7 @@ async function extractInfrastructureNames(): Promise { console.log('⚠️ Warning: No environment directories found in infrastructure/server-setup/inventory/'); return []; } - console.log('List of existing infrastructure configurations:'); - console.log(infraEnvironments.join(', ')); - + log('🔍 Found infrastructure configurations:', infraEnvironments.join(', ')); return infraEnvironments; } @@ -37,9 +35,7 @@ async function extractEnvironmentNames(): Promise { return []; } - console.log('\nList of existing environment configurations:'); - console.log(environments.join(', ')); - + log('🔍 Found OpenCRVS configurations:', environments.join(', ')); return environments; } @@ -72,8 +68,6 @@ async function updateWorkflows( const { workflows } = config; for (const workflowPath of workflows) { - console.log(`\nUpdating ${workflowPath} with: [${envList.join(', ')}]`); - try { const fileContents = readFileSync(workflowPath, 'utf8'); @@ -87,8 +81,9 @@ async function updateWorkflows( const updatedContent = updateOptionsInYaml(fileContents, envList); writeFileSync(workflowPath, updatedContent, 'utf8'); - console.log(`✓ Successfully updated ${workflowPath}`); + log(` ✓ Successfully updated ${workflowPath}`); } catch (error) { + console.error(`\n⚠️ Error updating ${workflowPath} with environments: [${envList.join(', ')}]`); console.error(`✗ Failed to update ${workflowPath}:`, error); throw error; } @@ -96,16 +91,12 @@ async function updateWorkflows( } export async function updateWorkflowEnvironments(): Promise { - try { - console.log('🔄 Updating workflow environments...\n'); - + try { // Extract infrastructure names const infraEnvironments = await extractInfrastructureNames(); - // Extract environment names (only directories) - const environments = await extractEnvironmentNames(); - // Update workflows with infrastructure configurations + console.log('🔄 Updating infrastructure workflows:'); await updateWorkflows(infraEnvironments, { workflows: [ '.github/workflows/provision.yml', @@ -114,7 +105,9 @@ export async function updateWorkflowEnvironments(): Promise { path: 'on.workflow_dispatch.inputs.environment.options' }); - console.log(`\n📋 Updating workflows...`); + // Extract environment names (only directories) + const environments = await extractEnvironmentNames(); + const workflows = [ '.github/workflows/deploy-dependencies.yml', '.github/workflows/deploy-opencrvs.yml', @@ -123,16 +116,14 @@ export async function updateWorkflowEnvironments(): Promise { '.github/workflows/k8s-reindex.yml', '.github/workflows/github-to-k8s-sync-env.yml' ]; - + log("📋 Updating OpenCRVS application workflows:"); await updateWorkflows(environments, { workflows, path: 'on.workflow_dispatch.inputs.environment.options' }); - console.log('\n✅ All workflows updated successfully!'); - console.log('\n💡 Review the changes and commit them when ready.'); - + success('✅ All workflows updated successfully!'); } catch (error) { console.error('\n❌ Error updating workflows:', error); process.exit(1); diff --git a/infrastructure/server-setup/tasks/k8s/install-containerd.yml b/infrastructure/server-setup/tasks/k8s/install-containerd.yml index 6f911c11..e1ffaa73 100644 --- a/infrastructure/server-setup/tasks/k8s/install-containerd.yml +++ b/infrastructure/server-setup/tasks/k8s/install-containerd.yml @@ -5,7 +5,13 @@ purge: true loop: - docker + - docker-ce + - docker-ce-cli + - docker-buildx-plugin + - docker-ce-rootless-extras + - docker-compose-plugin - docker-engine + - python3-docker - docker.io - containerd - runc diff --git a/package.json b/package.json index e33c87d3..237630df 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "scripts": { "environment:init": "ts-node infrastructure/environments/setup-environment.ts", "environment:upgrade": "yarn environment:init", + "environment:swarm-to-k8s": "ts-node infrastructure/environments/swarm-to-k8s.ts", "prepare": "husky" }, "devDependencies": { @@ -28,7 +29,8 @@ "ts-node": "^10.9.1", "typescript": "^5.1.6", "js-yaml": "4.1.0", - "glob": "11.0.3" + "glob": "11.0.3", + "handlebars": "^4.7.8" }, "dependencies": { "@types/node": "^24.0.0", diff --git a/yarn.lock b/yarn.lock index 34f2ff28..d6af4c7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -338,6 +338,18 @@ glob@11.0.3: package-json-from-dist "^1.0.0" path-scurry "^2.0.0" +handlebars@^4.7.8: + version "4.7.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.8.tgz#41c42c18b1be2365439188c77c6afae71c0cd9e9" + integrity sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ== + dependencies: + minimist "^1.2.5" + neo-async "^2.6.2" + source-map "^0.6.1" + wordwrap "^1.0.0" + optionalDependencies: + uglify-js "^3.1.4" + husky@9.1.7: version "9.1.7" resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" @@ -411,6 +423,11 @@ minimatch@^10.0.3: dependencies: "@isaacs/brace-expansion" "^5.0.0" +minimist@^1.2.5: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + minipass@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" @@ -421,6 +438,11 @@ nan@^2.19.0, nan@^2.23.0: resolved "https://registry.yarnpkg.com/nan/-/nan-2.24.0.tgz#a8919b36e692aa5b260831910e4f81419fc0a283" integrity sha512-Vpf9qnVW1RaDkoNKFUvfxqAbtI8ncb8OJlqZ9wwpXzWPEsvsB1nvdUi6oYrHIkQ1Y/tMDnr1h4nczS0VB9Xykg== +neo-async@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" + integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== + node-fetch@^2.6.7: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -488,6 +510,11 @@ sisteransi@^1.0.5: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== +source-map@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + ssh2@^1.17.0: version "1.17.0" resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-1.17.0.tgz#dc686e8e3abdbd4ad95d46fa139615903c12258c" @@ -581,6 +608,11 @@ typescript@^5.1.6: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.9.2.tgz#d93450cddec5154a2d5cabe3b8102b83316fb2a6" integrity sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A== +uglify-js@^3.1.4: + version "3.19.3" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.19.3.tgz#82315e9bbc6f2b25888858acd1fff8441035b77f" + integrity sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ== + undici-types@~5.26.4: version "5.26.5" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" @@ -626,6 +658,11 @@ which@^2.0.1: dependencies: isexe "^2.0.0" +wordwrap@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" + integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"