From bb031309ac824cb4d341f7dc8abea6fb11b328e7 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Fri, 16 Jan 2026 21:19:38 -0500 Subject: [PATCH 01/11] Initial script --- packages/db/scripts/seed_devdb.ts | 189 ++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 packages/db/scripts/seed_devdb.ts diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts new file mode 100644 index 00000000..7b6a0570 --- /dev/null +++ b/packages/db/scripts/seed_devdb.ts @@ -0,0 +1,189 @@ +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type { Client } from "pg"; +import { sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import Pool from "pg-pool"; + +import { isDiscordAdmin } from "../../api/src/utils"; +import { env } from "../src/env"; +import * as authSchema from "../src/schemas/auth"; +import * as knightHacksSchema from "../src/schemas/knight-hacks"; + +/* eslint-disable no-console */ +// Usage: +// pnpm --filter @forge/db with-env tsx scripts/seed_devdb.ts + +console.log("Starting seeding script"); + +type AuthSchema = typeof authSchema; +type KnightHacksSchema = typeof knightHacksSchema; +type DatabaseSchema = AuthSchema & KnightHacksSchema; + +const backupDbName = `backup`; + +// Admin connection to postgres database for DDL operations +const adminPool = new Pool({ + connectionString: env.DATABASE_URL, + database: "postgres", +}); + +let backupPool: Pool | null = null; +let backupDb: NodePgDatabase | null = null; + +async function cleanUp() { + console.log("Cleaning up connections..."); + + if (backupPool !== null) { + try { + await backupPool.end(); + } catch (e) { + console.error("Error ending backup pool:", e); + } + } + + try { + await adminPool.end(); + } catch (e) { + console.error("Error ending admin pool:", e); + } +} + +const TABLES_REMOVE_ALL: string[] = [ + "auth_verification", + "auth_session", + "knight_hacks_judged_submission", + "knight_hacks_judges", +]; + +async function cleanTable(name: string, userIdsToDelete: string[]) { + if (!backupDb) return; + + if (TABLES_REMOVE_ALL.includes(name)) { + console.log("Removing all rows from ", name); + await backupDb.execute( + sql.raw(`TRUNCATE TABLE "${name}" RESTART IDENTITY CASCADE`), + ); + return; + } + + if (userIdsToDelete.length == 0) return; + + const relResult = await backupDb.execute(sql` + SELECT kcu.column_name + FROM information_schema.key_column_usage AS kcu + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = kcu.constraint_name + WHERE kcu.table_name = ${name} + AND ccu.table_name = 'auth_user' + AND ccu.column_name = 'id' + AND kcu.table_schema = 'public' + LIMIT 1; + `); + + const userFkColumn = relResult.rows[0]?.column_name as string | undefined; + + if (userFkColumn) { + console.log(`Cascading removing user info in table ${name}`); + + await backupDb.execute(sql` + DELETE FROM ${sql.identifier(name)} + WHERE ${sql.identifier(userFkColumn)} IN (${sql.join( + userIdsToDelete.map((id) => sql`${id}`), + sql`, `, + )}) + `); + } else { + console.log(`Table ${name} will remain unaffected entirely`); + } +} + +const isAdmin: Record = {}; + +async function shouldKeepId(userid: string): Promise { + if (!Object.keys(isAdmin).includes(userid)) { + const session = await backupDb?.query.User.findFirst({ + where: (t, { eq }) => eq(t.id, userid), + }); + if (!session) return false; + const admin = await isDiscordAdmin(session); + isAdmin[userid] = admin; + } + console.log(userid, isAdmin[userid]); + return isAdmin[userid] ?? false; +} + +async function main() { + try { + const baseConnectionString = env.DATABASE_URL.substring( + 0, + env.DATABASE_URL.lastIndexOf("/") + 1, + ); + + console.log(`Dropping database ${backupDbName} if it exists...`); + await adminPool.query(`DROP DATABASE IF EXISTS ${backupDbName}`); + + console.log(`Creating fresh database ${backupDbName}...`); + + await adminPool.query(` + CREATE DATABASE ${backupDbName} WITH TEMPLATE ${env.DATABASE_URL.split("/").pop()} + `); + + backupPool = new Pool({ + connectionString: baseConnectionString + backupDbName, + }); + + backupDb = drizzle({ + client: backupPool, + schema: { ...authSchema, ...knightHacksSchema }, + casing: "snake_case", + }); + + const { rows: tablesJSON } = await backupDb.execute(sql` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' +`); + + let tables = tablesJSON.map((t) => t.table_name as string); + tables = [...tables.filter((x) => x !== "auth_user"), "auth_user"]; + + const allUsers = await backupDb.execute(sql`SELECT id FROM auth_user`); + + const userIdsToDelete: string[] = []; + + for (const row of allUsers.rows) { + const userId = row.id as string; + const keep = await shouldKeepId(userId); + if (!keep) userIdsToDelete.push(userId); + } + + for (const tableName of tables) { + await cleanTable(tableName, userIdsToDelete); + } + + if (userIdsToDelete.length > 0) { + console.log( + `Deleting ${userIdsToDelete.length} non-admin users and cascading to whatever has cascade in the schema already`, + ); + + await backupDb.execute(sql` + DELETE FROM auth_user + WHERE id IN (${sql.join( + userIdsToDelete.map((id) => sql`${id}`), + sql`, `, + )}) + `); + } + + await cleanUp(); + + process.exit(0); + } catch (error) { + console.error("Error during database seeding:", error); + await cleanUp(); + process.exit(1); + } +} + +main(); From a49b14fcf42178859d45d4628b1732075bf9a01c Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Fri, 16 Jan 2026 21:26:39 -0500 Subject: [PATCH 02/11] diff copy --- packages/db/scripts/seed_devdb.ts | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 7b6a0570..15483a65 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -4,6 +4,12 @@ import { sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import Pool from "pg-pool"; +import { exec } from 'child_process'; +import { promisify } from 'util'; +import { unlink } from 'fs/promises'; + +const execAsync = promisify(exec); + import { isDiscordAdmin } from "../../api/src/utils"; import { env } from "../src/env"; import * as authSchema from "../src/schemas/auth"; @@ -112,6 +118,22 @@ async function shouldKeepId(userid: string): Promise { return isAdmin[userid] ?? false; } +async function copyDatabase(originalDb: string, newDb: string, password: string, user = 'root') { + const backupFile = 'backup.sql'; + const envN = { ...process.env, PGPASSWORD: password }; + + const host = "localhost"; + const port = 5432; + try { + await execAsync(`pg_dump -h ${host} -p ${port} -U ${user} ${originalDb} > ${backupFile}`, { env: envN }); + await execAsync(`createdb -h ${host} -p ${port} -U ${user} ${newDb}`, { env: envN }); + await execAsync(`psql -h ${host} -p ${port} -U ${user} ${newDb} < ${backupFile}`, { env: envN }); + + } finally { + await unlink(backupFile); + } +} + async function main() { try { const baseConnectionString = env.DATABASE_URL.substring( @@ -123,10 +145,7 @@ async function main() { await adminPool.query(`DROP DATABASE IF EXISTS ${backupDbName}`); console.log(`Creating fresh database ${backupDbName}...`); - - await adminPool.query(` - CREATE DATABASE ${backupDbName} WITH TEMPLATE ${env.DATABASE_URL.split("/").pop()} - `); + await copyDatabase("local", backupDbName, "mysecretpassword"); backupPool = new Pool({ connectionString: baseConnectionString + backupDbName, From 0ac25f134a637f69a326a7f8f7ebae768d8184f9 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Fri, 16 Jan 2026 23:43:43 -0500 Subject: [PATCH 03/11] moved auth to local blade perm instead of disc and url parsing --- packages/db/scripts/seed_devdb.ts | 46 +++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 15483a65..bfb9b71d 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -1,6 +1,6 @@ import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Client } from "pg"; -import { sql } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import Pool from "pg-pool"; @@ -10,10 +10,11 @@ import { unlink } from 'fs/promises'; const execAsync = promisify(exec); -import { isDiscordAdmin } from "../../api/src/utils"; import { env } from "../src/env"; import * as authSchema from "../src/schemas/auth"; import * as knightHacksSchema from "../src/schemas/knight-hacks"; +import { Roles } from "../src/schemas/auth"; +import { Permissions } from "../src/schemas/auth"; /* eslint-disable no-console */ // Usage: @@ -57,6 +58,7 @@ async function cleanUp() { const TABLES_REMOVE_ALL: string[] = [ "auth_verification", "auth_session", + "auth_judge_session", "knight_hacks_judged_submission", "knight_hacks_judges", ]; @@ -107,33 +109,48 @@ const isAdmin: Record = {}; async function shouldKeepId(userid: string): Promise { if (!Object.keys(isAdmin).includes(userid)) { - const session = await backupDb?.query.User.findFirst({ - where: (t, { eq }) => eq(t.id, userid), - }); - if (!session) return false; - const admin = await isDiscordAdmin(session); - isAdmin[userid] = admin; + if(!backupDb) return false; + const permRows = await backupDb + .select({ + permissions: Roles.permissions, + }) + .from(Roles) + .innerJoin(Permissions, eq(Roles.id, Permissions.roleId)) + .where(sql`cast(${Permissions.userId} as text) = ${userid}`); + + const admin = permRows.length > 0; + isAdmin[userid] = admin; } console.log(userid, isAdmin[userid]); return isAdmin[userid] ?? false; } -async function copyDatabase(originalDb: string, newDb: string, password: string, user = 'root') { +async function copyDatabase() { const backupFile = 'backup.sql'; + const { originalDb, user, password, host, port } = parsePg(); const envN = { ...process.env, PGPASSWORD: password }; - const host = "localhost"; - const port = 5432; try { await execAsync(`pg_dump -h ${host} -p ${port} -U ${user} ${originalDb} > ${backupFile}`, { env: envN }); - await execAsync(`createdb -h ${host} -p ${port} -U ${user} ${newDb}`, { env: envN }); - await execAsync(`psql -h ${host} -p ${port} -U ${user} ${newDb} < ${backupFile}`, { env: envN }); + await execAsync(`createdb -h ${host} -p ${port} -U ${user} ${backupDbName}`, { env: envN }); + await execAsync(`psql -h ${host} -p ${port} -U ${user} ${backupDbName} < ${backupFile}`, { env: envN }); } finally { await unlink(backupFile); } } +function parsePg() { + const u = new URL(env.DATABASE_URL); + return { + originalDb: u.pathname.slice(1), + user: u.username, + password: u.password, + host: u.hostname, + port: u.port + }; +} + async function main() { try { const baseConnectionString = env.DATABASE_URL.substring( @@ -145,7 +162,7 @@ async function main() { await adminPool.query(`DROP DATABASE IF EXISTS ${backupDbName}`); console.log(`Creating fresh database ${backupDbName}...`); - await copyDatabase("local", backupDbName, "mysecretpassword"); + await copyDatabase(); backupPool = new Pool({ connectionString: baseConnectionString + backupDbName, @@ -206,3 +223,4 @@ async function main() { } main(); + From 4521471403b79891f39dd2880611edf6cc7f0533 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Fri, 16 Jan 2026 23:44:06 -0500 Subject: [PATCH 04/11] format --- packages/db/scripts/seed_devdb.ts | 66 +++++++++++++++++-------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index bfb9b71d..83b8b527 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -1,20 +1,19 @@ +import { exec } from "child_process"; +import { unlink } from "fs/promises"; +import { promisify } from "util"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Client } from "pg"; import { eq, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import Pool from "pg-pool"; -import { exec } from 'child_process'; -import { promisify } from 'util'; -import { unlink } from 'fs/promises'; - -const execAsync = promisify(exec); - import { env } from "../src/env"; import * as authSchema from "../src/schemas/auth"; -import * as knightHacksSchema from "../src/schemas/knight-hacks"; import { Roles } from "../src/schemas/auth"; import { Permissions } from "../src/schemas/auth"; +import * as knightHacksSchema from "../src/schemas/knight-hacks"; + +const execAsync = promisify(exec); /* eslint-disable no-console */ // Usage: @@ -58,7 +57,7 @@ async function cleanUp() { const TABLES_REMOVE_ALL: string[] = [ "auth_verification", "auth_session", - "auth_judge_session", + "auth_judge_session", "knight_hacks_judged_submission", "knight_hacks_judges", ]; @@ -109,32 +108,40 @@ const isAdmin: Record = {}; async function shouldKeepId(userid: string): Promise { if (!Object.keys(isAdmin).includes(userid)) { - if(!backupDb) return false; - const permRows = await backupDb - .select({ - permissions: Roles.permissions, - }) - .from(Roles) - .innerJoin(Permissions, eq(Roles.id, Permissions.roleId)) - .where(sql`cast(${Permissions.userId} as text) = ${userid}`); - - const admin = permRows.length > 0; - isAdmin[userid] = admin; + if (!backupDb) return false; + const permRows = await backupDb + .select({ + permissions: Roles.permissions, + }) + .from(Roles) + .innerJoin(Permissions, eq(Roles.id, Permissions.roleId)) + .where(sql`cast(${Permissions.userId} as text) = ${userid}`); + + const admin = permRows.length > 0; + isAdmin[userid] = admin; } console.log(userid, isAdmin[userid]); return isAdmin[userid] ?? false; } async function copyDatabase() { - const backupFile = 'backup.sql'; - const { originalDb, user, password, host, port } = parsePg(); + const backupFile = "backup.sql"; + const { originalDb, user, password, host, port } = parsePg(); const envN = { ...process.env, PGPASSWORD: password }; - + try { - await execAsync(`pg_dump -h ${host} -p ${port} -U ${user} ${originalDb} > ${backupFile}`, { env: envN }); - await execAsync(`createdb -h ${host} -p ${port} -U ${user} ${backupDbName}`, { env: envN }); - await execAsync(`psql -h ${host} -p ${port} -U ${user} ${backupDbName} < ${backupFile}`, { env: envN }); - + await execAsync( + `pg_dump -h ${host} -p ${port} -U ${user} ${originalDb} > ${backupFile}`, + { env: envN }, + ); + await execAsync( + `createdb -h ${host} -p ${port} -U ${user} ${backupDbName}`, + { env: envN }, + ); + await execAsync( + `psql -h ${host} -p ${port} -U ${user} ${backupDbName} < ${backupFile}`, + { env: envN }, + ); } finally { await unlink(backupFile); } @@ -146,8 +153,8 @@ function parsePg() { originalDb: u.pathname.slice(1), user: u.username, password: u.password, - host: u.hostname, - port: u.port + host: u.hostname, + port: u.port, }; } @@ -162,7 +169,7 @@ async function main() { await adminPool.query(`DROP DATABASE IF EXISTS ${backupDbName}`); console.log(`Creating fresh database ${backupDbName}...`); - await copyDatabase(); + await copyDatabase(); backupPool = new Pool({ connectionString: baseConnectionString + backupDbName, @@ -223,4 +230,3 @@ async function main() { } main(); - From 4e7f2711ac352198f9d96d363f7e9efce1e1df7d Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Sat, 17 Jan 2026 10:16:20 -0500 Subject: [PATCH 05/11] Faster getting admins --- packages/db/scripts/seed_devdb.ts | 77 ++++++++++--------------------- 1 file changed, 25 insertions(+), 52 deletions(-) diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 83b8b527..a07f8526 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -3,14 +3,12 @@ import { unlink } from "fs/promises"; import { promisify } from "util"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Client } from "pg"; -import { eq, sql } from "drizzle-orm"; +import { sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import Pool from "pg-pool"; import { env } from "../src/env"; import * as authSchema from "../src/schemas/auth"; -import { Roles } from "../src/schemas/auth"; -import { Permissions } from "../src/schemas/auth"; import * as knightHacksSchema from "../src/schemas/knight-hacks"; const execAsync = promisify(exec); @@ -62,7 +60,7 @@ const TABLES_REMOVE_ALL: string[] = [ "knight_hacks_judges", ]; -async function cleanTable(name: string, userIdsToDelete: string[]) { +async function cleanTable(name: string, userIdsToKeep: string[]) { if (!backupDb) return; if (TABLES_REMOVE_ALL.includes(name)) { @@ -73,8 +71,6 @@ async function cleanTable(name: string, userIdsToDelete: string[]) { return; } - if (userIdsToDelete.length == 0) return; - const relResult = await backupDb.execute(sql` SELECT kcu.column_name FROM information_schema.key_column_usage AS kcu @@ -94,8 +90,8 @@ async function cleanTable(name: string, userIdsToDelete: string[]) { await backupDb.execute(sql` DELETE FROM ${sql.identifier(name)} - WHERE ${sql.identifier(userFkColumn)} IN (${sql.join( - userIdsToDelete.map((id) => sql`${id}`), + WHERE ${sql.identifier(userFkColumn)} NOT IN (${sql.join( + userIdsToKeep.map((id) => sql`${id}`), sql`, `, )}) `); @@ -104,26 +100,6 @@ async function cleanTable(name: string, userIdsToDelete: string[]) { } } -const isAdmin: Record = {}; - -async function shouldKeepId(userid: string): Promise { - if (!Object.keys(isAdmin).includes(userid)) { - if (!backupDb) return false; - const permRows = await backupDb - .select({ - permissions: Roles.permissions, - }) - .from(Roles) - .innerJoin(Permissions, eq(Roles.id, Permissions.roleId)) - .where(sql`cast(${Permissions.userId} as text) = ${userid}`); - - const admin = permRows.length > 0; - isAdmin[userid] = admin; - } - console.log(userid, isAdmin[userid]); - return isAdmin[userid] ?? false; -} - async function copyDatabase() { const backupFile = "backup.sql"; const { originalDb, user, password, host, port } = parsePg(); @@ -182,42 +158,39 @@ async function main() { }); const { rows: tablesJSON } = await backupDb.execute(sql` - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_type = 'BASE TABLE' -`); + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + `); let tables = tablesJSON.map((t) => t.table_name as string); tables = [...tables.filter((x) => x !== "auth_user"), "auth_user"]; - const allUsers = await backupDb.execute(sql`SELECT id FROM auth_user`); - - const userIdsToDelete: string[] = []; - - for (const row of allUsers.rows) { - const userId = row.id as string; - const keep = await shouldKeepId(userId); - if (!keep) userIdsToDelete.push(userId); - } + console.log("Getting admins"); + const userIdsToKeep: string[] = ( + await backupDb.query.Permissions.findMany({ + columns: { + userId: true, + }, + }) + ).map((t) => t.userId); for (const tableName of tables) { - await cleanTable(tableName, userIdsToDelete); + await cleanTable(tableName, userIdsToKeep); } - if (userIdsToDelete.length > 0) { - console.log( - `Deleting ${userIdsToDelete.length} non-admin users and cascading to whatever has cascade in the schema already`, - ); + console.log( + `Keeping ${userIdsToKeep.length} non-admin users and cascading to whatever has cascade in the schema already`, + ); - await backupDb.execute(sql` + await backupDb.execute(sql` DELETE FROM auth_user - WHERE id IN (${sql.join( - userIdsToDelete.map((id) => sql`${id}`), + WHERE id NOT IN (${sql.join( + userIdsToKeep.map((id) => sql`${id}`), sql`, `, )}) `); - } await cleanUp(); @@ -229,4 +202,4 @@ async function main() { } } -main(); +await main(); From fea778acd46bc9d943b3cb52b2f9e01d264a2168 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Sun, 18 Jan 2026 23:37:10 -0500 Subject: [PATCH 06/11] minio and discord role syncing --- packages/db/scripts/seed_devdb.ts | 227 ++++++++++++++++++++++++++++-- 1 file changed, 218 insertions(+), 9 deletions(-) diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index a07f8526..719d1923 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -1,17 +1,26 @@ import { exec } from "child_process"; import { unlink } from "fs/promises"; import { promisify } from "util"; +const execAsync = promisify(exec); import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Client } from "pg"; +import { Routes } from "discord-api-types/v10"; import { sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/node-postgres"; import Pool from "pg-pool"; +import { + DEV_KNIGHTHACKS_GUILD_ID, + KNIGHTHACKS_S3_BUCKET_REGION, + PROD_KNIGHTHACKS_GUILD_ID, +} from "@forge/consts/knight-hacks"; + +import { discord } from "../../api/src/utils"; import { env } from "../src/env"; import * as authSchema from "../src/schemas/auth"; import * as knightHacksSchema from "../src/schemas/knight-hacks"; +import { minioClient } from "../../api/src/minio/minio-client"; -const execAsync = promisify(exec); /* eslint-disable no-console */ // Usage: @@ -60,6 +69,9 @@ const TABLES_REMOVE_ALL: string[] = [ "knight_hacks_judges", ]; +const roleIdMappings: Record = {}; +const eventIdMappings: Record = {}; + async function cleanTable(name: string, userIdsToKeep: string[]) { if (!backupDb) return; @@ -95,6 +107,29 @@ async function cleanTable(name: string, userIdsToKeep: string[]) { sql`, `, )}) `); + } else if (name === "auth_roles") { + const all_rows = await backupDb.query.Roles.findMany(); + for (const row of all_rows) { + const id = row.id; + await backupDb.execute(sql` + UPDATE "auth_roles" + SET discord_role_id = ${roleIdMappings[row.discordRoleId]} + WHERE id = ${id} + `); + } + console.log("Updated auth roles to use dev server discord ids"); + } else if (name === "knight_hacks_event") { + const all_rows = await backupDb.query.Event.findMany(); + for (const row of all_rows) { + if (!eventIdMappings[row.discordId]) continue; + const id = row.id; + await backupDb.execute(sql` + UPDATE "knight_hacks_event" + SET discord_id = ${eventIdMappings[row.discordId]} + WHERE id = ${id} + `); + } + console.log("Updated KH events to use dev server discord ids"); } else { console.log(`Table ${name} will remain unaffected entirely`); } @@ -103,6 +138,7 @@ async function cleanTable(name: string, userIdsToKeep: string[]) { async function copyDatabase() { const backupFile = "backup.sql"; const { originalDb, user, password, host, port } = parsePg(); + /* eslint-disable no-restricted-properties */ const envN = { ...process.env, PGPASSWORD: password }; try { @@ -134,6 +170,174 @@ function parsePg() { }; } +interface RoleColors { + primary_color: number; + secondary_color: number | null; + tertiary_color: number | null; +} + +interface DiscordRole { + id: string; + name: string; + description: string | null; + permissions: string; + position: number; + color: number; + colors: RoleColors; + hoist: boolean; + managed: boolean; + mentionable: boolean; + icon: string | null; + unicode_emoji: string | null; + flags: number; +} + +async function syncRoles() { + if (!backupDb) return; + + const prodRolesWithPerms = new Set( + ( + await backupDb.query.Roles.findMany({ columns: { discordRoleId: true } }) + ).map((row) => row.discordRoleId), + ); + let prodRoles = (await discord.get( + Routes.guildRoles(PROD_KNIGHTHACKS_GUILD_ID), + )) as DiscordRole[]; + prodRoles = prodRoles.filter((role) => prodRolesWithPerms.has(role.id)); + + const devRolesArr = (await discord.get( + Routes.guildRoles(DEV_KNIGHTHACKS_GUILD_ID), + )) as DiscordRole[]; + const devRoles = Object.fromEntries( + devRolesArr.map((role) => [role.name + " " + role.permissions, role]), + ); + + for (const role of prodRoles) { + const hash = role.name + " " + role.permissions; + if (devRoles[hash]) { + roleIdMappings[role.id] = devRoles[hash].id; + } else { + const newRole = (await discord.post( + Routes.guildRoles(DEV_KNIGHTHACKS_GUILD_ID), + { + body: { + name: role.name, + permissions: role.permissions, + color: role.color, + hoist: role.hoist, + mentionable: role.mentionable, + }, + }, + )) as DiscordRole; + roleIdMappings[role.id] = newRole.id; + } + } + + console.log(roleIdMappings); +} + +interface DiscordGuildScheduledEvent { + id: string; + guild_id: string; + channel_id: string | null; + name: string; + description: string | null; + scheduled_start_time: string; + scheduled_end_time: string | null; + privacy_level: number; + status: number; + entity_type: number; + entity_id: string | null; + entity_metadata: { + location?: string; + } | null; + creator_id?: string; + /* eslint-disable @typescript-eslint/no-explicit-any */ + creator?: any; + user_count?: number; + image?: string | null; +} + +async function syncEvents() { + if (!backupDb) return; + + const prodEvents = (await discord.get( + Routes.guildScheduledEvents(PROD_KNIGHTHACKS_GUILD_ID), + )) as DiscordGuildScheduledEvent[]; + + const devEventsArr = (await discord.get( + Routes.guildScheduledEvents(DEV_KNIGHTHACKS_GUILD_ID), + )) as DiscordGuildScheduledEvent[]; + const devEvents = Object.fromEntries( + devEventsArr.map((ev) => [ev.name + " " + ev.scheduled_start_time, ev]), + ); + + for (const event of prodEvents) { + const hash = event.name + " " + event.scheduled_start_time; + if (devEvents[hash]) { + eventIdMappings[event.id] = devEvents[hash].id; + } else { + const newEvent = (await discord.post( + Routes.guildScheduledEvents(DEV_KNIGHTHACKS_GUILD_ID), + { + body: { + name: event.name, + description: event.description, + scheduled_start_time: event.scheduled_start_time, + scheduled_end_time: event.scheduled_end_time, + privacy_level: event.privacy_level, + entity_type: event.entity_type, + entity_metadata: event.entity_metadata, + }, + }, + )) as DiscordGuildScheduledEvent; + eventIdMappings[event.id] = newEvent.id; + } + } + + console.log(eventIdMappings); +} + +async function minio() { + const BUCKET_NAME = "dev-db-backups"; + const filePath = "backup.sql"; + const { originalDb: _originalDb, user, password, host, port } = parsePg(); + /* eslint-disable no-restricted-properties */ + const envN = { ...process.env, PGPASSWORD: password }; + + await execAsync( + `pg_dump -h ${host} -p ${port} -U ${user} --data-only --column-inserts --disable-triggers --no-owner --no-acl ${backupDbName} > ${filePath}`, + { env: envN }, + ); + + const bucketExists = await minioClient.bucketExists(BUCKET_NAME); + if (!bucketExists) { + await minioClient.makeBucket( + BUCKET_NAME, + KNIGHTHACKS_S3_BUCKET_REGION, + ); + } + + await minioClient.fPutObject( + BUCKET_NAME, + filePath, + filePath, + { + "Content-Type": "text/plain", + } + ); + + const fileUrl = await minioClient.presignedGetObject( + BUCKET_NAME, + filePath, + 60 * 60 * 24, + ); + + await unlink(filePath); + + console.log(`Saved backup to ${fileUrl}`); +} + async function main() { try { const baseConnectionString = env.DATABASE_URL.substring( @@ -157,6 +361,9 @@ async function main() { casing: "snake_case", }); + await syncRoles(); + await syncEvents(); + const { rows: tablesJSON } = await backupDb.execute(sql` SELECT table_name FROM information_schema.tables @@ -167,7 +374,7 @@ async function main() { let tables = tablesJSON.map((t) => t.table_name as string); tables = [...tables.filter((x) => x !== "auth_user"), "auth_user"]; - console.log("Getting admins"); + console.log("Getting admins"); const userIdsToKeep: string[] = ( await backupDb.query.Permissions.findMany({ columns: { @@ -181,16 +388,18 @@ async function main() { } console.log( - `Keeping ${userIdsToKeep.length} non-admin users and cascading to whatever has cascade in the schema already`, + `Keeping ${userIdsToKeep.length} admin users and cascading to whatever has cascade in the schema already`, ); await backupDb.execute(sql` - DELETE FROM auth_user - WHERE id NOT IN (${sql.join( - userIdsToKeep.map((id) => sql`${id}`), - sql`, `, - )}) - `); + DELETE FROM auth_user + WHERE id NOT IN (${sql.join( + userIdsToKeep.map((id) => sql`${id}`), + sql`, `, + )}) + `); + + await minio(); await cleanUp(); From bc170ad68035185cc2e8b65a2318f495a00839a3 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Sun, 18 Jan 2026 23:45:44 -0500 Subject: [PATCH 07/11] pulling and seeding script + dropping backup db --- packages/db/scripts/get_prod_db.ts | 62 ++++++++++++++++++++++++++++++ packages/db/scripts/seed_devdb.ts | 10 ++++- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 packages/db/scripts/get_prod_db.ts diff --git a/packages/db/scripts/get_prod_db.ts b/packages/db/scripts/get_prod_db.ts new file mode 100644 index 00000000..d2795f3c --- /dev/null +++ b/packages/db/scripts/get_prod_db.ts @@ -0,0 +1,62 @@ +/* eslint-disable no-console */ +// Usage: +// pnpm --filter @forge/db with-env tsx scripts/get_prod_db.tsx + +import { exec } from "child_process"; +import { unlink } from "fs/promises"; +import { promisify } from "util"; +import { minioClient } from "../../api/src/minio/minio-client"; +import fs from "fs"; +import { pipeline } from "stream/promises"; +import { env } from "../src/env"; +const execAsync = promisify(exec); + +function parsePg() { + const u = new URL(env.DATABASE_URL); + return { + originalDb: u.pathname.slice(1), + user: u.username, + password: u.password, + host: u.hostname, + port: u.port, + }; +} + +async function main() { + const BUCKET_NAME = "dev-db-backups"; + const objectName = "backup.sql"; + + const fileUrl = await minioClient.presignedGetObject( + BUCKET_NAME, + objectName, + 60 * 60 * 24, + ); + + console.log("Pulling backup.sql from minio"); + + const res = await fetch(fileUrl); + if (!res.ok || !res.body) { + throw new Error(`Download failed: ${res.status}`); + } + + await pipeline( + res.body, + fs.createWriteStream(objectName), + ); + + const { originalDb: _, user, password, host: _host, port } = parsePg(); + /* eslint-disable no-restricted-properties */ + const envN = { ...process.env, PGPASSWORD: password }; + + console.log("Inserting prod rows into local DB"); + try { + await execAsync( + `psql -h localhost -p ${port} -U ${user} local < ${objectName}`, + { env: envN }, + ); + } finally { + await unlink(objectName); + } +} + +await main(); diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 719d1923..1d0e74ee 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -44,7 +44,7 @@ let backupPool: Pool | null = null; let backupDb: NodePgDatabase | null = null; async function cleanUp() { - console.log("Cleaning up connections..."); + console.log("Cleaning up connections"); if (backupPool !== null) { try { @@ -59,6 +59,14 @@ async function cleanUp() { } catch (e) { console.error("Error ending admin pool:", e); } + + const { originalDb: _, user, password, host, port } = parsePg(); + /* eslint-disable no-restricted-properties */ + const envN = { ...process.env, PGPASSWORD: password }; + await execAsync( + `dropdb -h ${host} -p ${port} -U ${user} backup`, + { env: envN }, + ); } const TABLES_REMOVE_ALL: string[] = [ From eeb2f539a00234611fef033e8afd3a7dddbd3b4f Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Mon, 19 Jan 2026 00:09:01 -0500 Subject: [PATCH 08/11] Added cron to make filtered backup --- apps/tk/src/hooks/backup-filtered-db.ts | 22 ++++++++++++++++++++++ apps/tk/src/hooks/index.ts | 2 ++ 2 files changed, 24 insertions(+) create mode 100644 apps/tk/src/hooks/backup-filtered-db.ts diff --git a/apps/tk/src/hooks/backup-filtered-db.ts b/apps/tk/src/hooks/backup-filtered-db.ts new file mode 100644 index 00000000..1fe15447 --- /dev/null +++ b/apps/tk/src/hooks/backup-filtered-db.ts @@ -0,0 +1,22 @@ +import cron from "node-cron"; +import { exec } from "child_process"; + +const COMMAND = "pnpm --filter @forge/db with-env tsx scripts/seed_devdb.ts"; + +export function execute() { + cron.schedule("0 8 * * *", () => { + console.log("[CRON] Prod DB backup for local dev job fired:", new Date().toISOString()); + + exec(COMMAND, (error, stdout) => { + if (error) { + console.error("[CRON] Command failed:", error); + return; + } + if (stdout) { + console.log("[CRON] Command stdout:", stdout); + } + }); + }); +} + + diff --git a/apps/tk/src/hooks/index.ts b/apps/tk/src/hooks/index.ts index ad0096b5..56850107 100644 --- a/apps/tk/src/hooks/index.ts +++ b/apps/tk/src/hooks/index.ts @@ -5,6 +5,7 @@ import { execute as daily } from "./daily"; import { execute as emailQueue } from "./email-queue"; import { execute as reminder } from "./reminder"; import { execute as roleSync } from "./role-sync"; +import { execute as prodBackup } from "./backup-filtered-db"; // Export all commands export const hooks = { @@ -15,4 +16,5 @@ export const hooks = { emailQueue, alumniSync, roleSync, + prodBackup }; From eadf4fd3b5e4d7ee6b770bbb86d7bf3f13239a71 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Mon, 19 Jan 2026 00:09:54 -0500 Subject: [PATCH 09/11] format --- apps/tk/src/hooks/backup-filtered-db.ts | 11 ++-- apps/tk/src/hooks/index.ts | 4 +- packages/db/scripts/get_prod_db.ts | 37 ++++++----- packages/db/scripts/seed_devdb.ts | 85 +++++++++++-------------- 4 files changed, 64 insertions(+), 73 deletions(-) diff --git a/apps/tk/src/hooks/backup-filtered-db.ts b/apps/tk/src/hooks/backup-filtered-db.ts index 1fe15447..d50d82d9 100644 --- a/apps/tk/src/hooks/backup-filtered-db.ts +++ b/apps/tk/src/hooks/backup-filtered-db.ts @@ -1,11 +1,14 @@ -import cron from "node-cron"; import { exec } from "child_process"; +import cron from "node-cron"; -const COMMAND = "pnpm --filter @forge/db with-env tsx scripts/seed_devdb.ts"; +const COMMAND = "pnpm --filter @forge/db with-env tsx scripts/seed_devdb.ts"; export function execute() { cron.schedule("0 8 * * *", () => { - console.log("[CRON] Prod DB backup for local dev job fired:", new Date().toISOString()); + console.log( + "[CRON] Prod DB backup for local dev job fired:", + new Date().toISOString(), + ); exec(COMMAND, (error, stdout) => { if (error) { @@ -18,5 +21,3 @@ export function execute() { }); }); } - - diff --git a/apps/tk/src/hooks/index.ts b/apps/tk/src/hooks/index.ts index 56850107..9b4fc026 100644 --- a/apps/tk/src/hooks/index.ts +++ b/apps/tk/src/hooks/index.ts @@ -1,11 +1,11 @@ // import { execute as beep } from "./beep"; import { execute as alumniSync } from "./alumni-assign"; import { execute as animals } from "./animals"; +import { execute as prodBackup } from "./backup-filtered-db"; import { execute as daily } from "./daily"; import { execute as emailQueue } from "./email-queue"; import { execute as reminder } from "./reminder"; import { execute as roleSync } from "./role-sync"; -import { execute as prodBackup } from "./backup-filtered-db"; // Export all commands export const hooks = { @@ -16,5 +16,5 @@ export const hooks = { emailQueue, alumniSync, roleSync, - prodBackup + prodBackup, }; diff --git a/packages/db/scripts/get_prod_db.ts b/packages/db/scripts/get_prod_db.ts index d2795f3c..3751fb7a 100644 --- a/packages/db/scripts/get_prod_db.ts +++ b/packages/db/scripts/get_prod_db.ts @@ -3,12 +3,14 @@ // pnpm --filter @forge/db with-env tsx scripts/get_prod_db.tsx import { exec } from "child_process"; +import fs from "fs"; import { unlink } from "fs/promises"; +import { pipeline } from "stream/promises"; import { promisify } from "util"; + import { minioClient } from "../../api/src/minio/minio-client"; -import fs from "fs"; -import { pipeline } from "stream/promises"; import { env } from "../src/env"; + const execAsync = promisify(exec); function parsePg() { @@ -23,32 +25,29 @@ function parsePg() { } async function main() { - const BUCKET_NAME = "dev-db-backups"; - const objectName = "backup.sql"; + const BUCKET_NAME = "dev-db-backups"; + const objectName = "backup.sql"; - const fileUrl = await minioClient.presignedGetObject( - BUCKET_NAME, - objectName, - 60 * 60 * 24, - ); + const fileUrl = await minioClient.presignedGetObject( + BUCKET_NAME, + objectName, + 60 * 60 * 24, + ); - console.log("Pulling backup.sql from minio"); + console.log("Pulling backup.sql from minio"); - const res = await fetch(fileUrl); - if (!res.ok || !res.body) { - throw new Error(`Download failed: ${res.status}`); - } + const res = await fetch(fileUrl); + if (!res.ok || !res.body) { + throw new Error(`Download failed: ${res.status}`); + } - await pipeline( - res.body, - fs.createWriteStream(objectName), - ); + await pipeline(res.body, fs.createWriteStream(objectName)); const { originalDb: _, user, password, host: _host, port } = parsePg(); /* eslint-disable no-restricted-properties */ const envN = { ...process.env, PGPASSWORD: password }; - console.log("Inserting prod rows into local DB"); + console.log("Inserting prod rows into local DB"); try { await execAsync( `psql -h localhost -p ${port} -U ${user} local < ${objectName}`, diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 1d0e74ee..6edbe3c1 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -1,7 +1,6 @@ import { exec } from "child_process"; import { unlink } from "fs/promises"; import { promisify } from "util"; -const execAsync = promisify(exec); import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Client } from "pg"; import { Routes } from "discord-api-types/v10"; @@ -15,12 +14,13 @@ import { PROD_KNIGHTHACKS_GUILD_ID, } from "@forge/consts/knight-hacks"; +import { minioClient } from "../../api/src/minio/minio-client"; import { discord } from "../../api/src/utils"; import { env } from "../src/env"; import * as authSchema from "../src/schemas/auth"; import * as knightHacksSchema from "../src/schemas/knight-hacks"; -import { minioClient } from "../../api/src/minio/minio-client"; +const execAsync = promisify(exec); /* eslint-disable no-console */ // Usage: @@ -60,13 +60,12 @@ async function cleanUp() { console.error("Error ending admin pool:", e); } - const { originalDb: _, user, password, host, port } = parsePg(); - /* eslint-disable no-restricted-properties */ - const envN = { ...process.env, PGPASSWORD: password }; - await execAsync( - `dropdb -h ${host} -p ${port} -U ${user} backup`, - { env: envN }, - ); + const { originalDb: _, user, password, host, port } = parsePg(); + /* eslint-disable no-restricted-properties */ + const envN = { ...process.env, PGPASSWORD: password }; + await execAsync(`dropdb -h ${host} -p ${port} -U ${user} backup`, { + env: envN, + }); } const TABLES_REMOVE_ALL: string[] = [ @@ -307,43 +306,35 @@ async function syncEvents() { } async function minio() { - const BUCKET_NAME = "dev-db-backups"; - const filePath = "backup.sql"; - const { originalDb: _originalDb, user, password, host, port } = parsePg(); - /* eslint-disable no-restricted-properties */ - const envN = { ...process.env, PGPASSWORD: password }; - - await execAsync( - `pg_dump -h ${host} -p ${port} -U ${user} --data-only --column-inserts --disable-triggers --no-owner --no-acl ${backupDbName} > ${filePath}`, - { env: envN }, - ); - - const bucketExists = await minioClient.bucketExists(BUCKET_NAME); - if (!bucketExists) { - await minioClient.makeBucket( - BUCKET_NAME, - KNIGHTHACKS_S3_BUCKET_REGION, - ); - } - - await minioClient.fPutObject( - BUCKET_NAME, - filePath, - filePath, - { - "Content-Type": "text/plain", - } - ); - - const fileUrl = await minioClient.presignedGetObject( - BUCKET_NAME, - filePath, - 60 * 60 * 24, - ); - - await unlink(filePath); - - console.log(`Saved backup to ${fileUrl}`); + const BUCKET_NAME = "dev-db-backups"; + const filePath = "backup.sql"; + const { originalDb: _originalDb, user, password, host, port } = parsePg(); + /* eslint-disable no-restricted-properties */ + const envN = { ...process.env, PGPASSWORD: password }; + + await execAsync( + `pg_dump -h ${host} -p ${port} -U ${user} --data-only --column-inserts --disable-triggers --no-owner --no-acl ${backupDbName} > ${filePath}`, + { env: envN }, + ); + + const bucketExists = await minioClient.bucketExists(BUCKET_NAME); + if (!bucketExists) { + await minioClient.makeBucket(BUCKET_NAME, KNIGHTHACKS_S3_BUCKET_REGION); + } + + await minioClient.fPutObject(BUCKET_NAME, filePath, filePath, { + "Content-Type": "text/plain", + }); + + const fileUrl = await minioClient.presignedGetObject( + BUCKET_NAME, + filePath, + 60 * 60 * 24, + ); + + await unlink(filePath); + + console.log(`Saved backup to ${fileUrl}`); } async function main() { @@ -407,7 +398,7 @@ async function main() { )}) `); - await minio(); + await minio(); await cleanUp(); From 0403e40d1a836a87e832b4641fb57865d8c89a46 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Mon, 19 Jan 2026 00:16:32 -0500 Subject: [PATCH 10/11] we love docs --- packages/db/scripts/get_prod_db.ts | 2 ++ packages/db/scripts/seed_devdb.ts | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/db/scripts/get_prod_db.ts b/packages/db/scripts/get_prod_db.ts index 3751fb7a..3d8c0ff2 100644 --- a/packages/db/scripts/get_prod_db.ts +++ b/packages/db/scripts/get_prod_db.ts @@ -2,6 +2,8 @@ // Usage: // pnpm --filter @forge/db with-env tsx scripts/get_prod_db.tsx +// Script to get prod db data into local db. Simply run the command above to get the rows and insert them automatically, this won't lose any data that you already have such as the superadmin from the bootstrap script. The bootstrap script can also be ran after running this command and it'll work fine. + import { exec } from "child_process"; import fs from "fs"; import { unlink } from "fs/promises"; diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 6edbe3c1..095bd690 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -1,3 +1,9 @@ +/* eslint-disable no-console */ +// Usage: +// pnpm --filter @forge/db with-env tsx scripts/seed_devdb.ts + +// A script to be run on prod only, this will take the prod db and make a backup sql script to insert all rows that don't have sensitive user data. It will only keep data from our admin members and delete any judging data/other sensitive data. It will also take all the server specific discord IDs in the DB and then sync them up with an event/role in the dev server and change the ID in the db for the local version. This sql file is uploaded to our minio client to be pulled by the get_prod_db.ts script. There's no realistic reason for this script to ever be ran on dev unless you're updating it cause I probably messed a lot up :D. See get_prod_db.ts for how to get prod data into your local db for deving. + import { exec } from "child_process"; import { unlink } from "fs/promises"; import { promisify } from "util"; @@ -21,11 +27,6 @@ import * as authSchema from "../src/schemas/auth"; import * as knightHacksSchema from "../src/schemas/knight-hacks"; const execAsync = promisify(exec); - -/* eslint-disable no-console */ -// Usage: -// pnpm --filter @forge/db with-env tsx scripts/seed_devdb.ts - console.log("Starting seeding script"); type AuthSchema = typeof authSchema; From 6ee8e86570d01a86a9d4eaa5d0f1d9100a1d00d4 Mon Sep 17 00:00:00 2001 From: DGoel1602 Date: Mon, 19 Jan 2026 18:13:57 -0500 Subject: [PATCH 11/11] review --- packages/db/scripts/seed_devdb.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/scripts/seed_devdb.ts b/packages/db/scripts/seed_devdb.ts index 095bd690..46191acc 100644 --- a/packages/db/scripts/seed_devdb.ts +++ b/packages/db/scripts/seed_devdb.ts @@ -225,6 +225,7 @@ async function syncRoles() { if (devRoles[hash]) { roleIdMappings[role.id] = devRoles[hash].id; } else { + await new Promise((resolve) => setTimeout(resolve, 100)); const newRole = (await discord.post( Routes.guildRoles(DEV_KNIGHTHACKS_GUILD_ID), { @@ -285,6 +286,7 @@ async function syncEvents() { if (devEvents[hash]) { eventIdMappings[event.id] = devEvents[hash].id; } else { + await new Promise((resolve) => setTimeout(resolve, 100)); const newEvent = (await discord.post( Routes.guildScheduledEvents(DEV_KNIGHTHACKS_GUILD_ID), {