From 853eacbb8be44174ad16c1d22f0d9b2bac36e643 Mon Sep 17 00:00:00 2001 From: DaxServer Date: Sat, 27 Sep 2025 18:34:09 +0200 Subject: [PATCH] feat: enable importing csv files --- backend/src/api/project/index.ts | 108 ++++++++++++------ backend/src/api/wikibase/index.ts | 30 ----- .../api/project/project.import-csv.test.ts | 76 ++++++++++++ .../pages/CreateProject.vue | 4 +- 4 files changed, 149 insertions(+), 69 deletions(-) create mode 100644 backend/tests/api/project/project.import-csv.test.ts diff --git a/backend/src/api/project/index.ts b/backend/src/api/project/index.ts index 97d81c5..7c681c4 100644 --- a/backend/src/api/project/index.ts +++ b/backend/src/api/project/index.ts @@ -99,7 +99,7 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) .post( '/import', - async ({ db, body: { name, file }, status }) => { + async ({ db, body: { name, file, hasHeaders = true }, status }) => { // Generate project name if not provided const projectName = name || generateProjectName(file.name) @@ -127,7 +127,11 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) // Save file to temporary location const timestamp = Date.now() const randomSuffix = Math.random().toString(36).substring(2, 8) - const tempFileName = `temp_${timestamp}_${randomSuffix}.json` + + // Determine file extension based on content type or file name + const isCSV = file.name.toLowerCase().endsWith('.csv') || file.type === 'text/csv' + const fileExtension = isCSV ? '.csv' : '.json' + const tempFileName = `temp_${timestamp}_${randomSuffix}${fileExtension}` const tempFilePath = `./temp/${tempFileName}` const fileBuffer = await file.arrayBuffer() @@ -135,26 +139,28 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) await Bun.write(tempFilePath, uint8Array) - // Check if JSON is parsable - try { - await Bun.file(tempFilePath).json() - } catch (parseError) { - await cleanupProject(db, project.id, tempFilePath) - - const errorMessage = - parseError instanceof Error ? parseError.message : 'Failed to parse JSON' - // Extract the specific error part from the message (e.g., "Unexpected identifier 'json'") - const match = errorMessage.match(/Unexpected identifier "([^"]+)"/) - const errorDetail = match - ? `JSON Parse error: Unexpected identifier "${match[1]}"` - : 'Failed to parse JSON' + // For JSON files, check if JSON is parsable + if (!isCSV) { + try { + await Bun.file(tempFilePath).json() + } catch (parseError) { + await cleanupProject(db, project.id, tempFilePath) + + const errorMessage = + parseError instanceof Error ? parseError.message : 'Failed to parse JSON' + // Extract the specific error part from the message (e.g., "Unexpected identifier 'json'") + const match = errorMessage.match(/Unexpected identifier "([^"]+)"/) + const errorDetail = match + ? `JSON Parse error: Unexpected identifier "${match[1]}"` + : 'Failed to parse JSON' - return status( - 400, - ApiErrorHandler.invalidJsonErrorWithData('Invalid JSON format in uploaded file', [ - errorDetail, - ]), - ) + return status( + 400, + ApiErrorHandler.invalidJsonErrorWithData('Invalid JSON format in uploaded file', [ + errorDetail, + ]), + ) + } } // Check if table already exists @@ -173,14 +179,24 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) ) } - // First, import the JSON data into the table - await db().run( - ` - CREATE TABLE "project_${project.id}" AS - SELECT * FROM read_json_auto(?) - `, - [tempFilePath], - ) + // Import the data into the table based on file type + if (isCSV) { + await db().run( + ` + CREATE TABLE "project_${project.id}" AS + SELECT * FROM read_csv(?, header = ?) + `, + [tempFilePath, hasHeaders], + ) + } else { + await db().run( + ` + CREATE TABLE "project_${project.id}" AS + SELECT * FROM read_json_auto(?) + `, + [tempFilePath], + ) + } // Check if 'id' column already exists and generate unique primary key column name const tableInfo = await db().runAndReadAll(`PRAGMA table_info("project_${project.id}")`) @@ -239,6 +255,9 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) error: 'Project name must be between 1 and 255 characters long if provided', }), ), + hasHeaders: t.Optional(t.BooleanString({ + default: true, + })), }), response: { 201: t.Object({ @@ -400,21 +419,32 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) // Try to create the table directly try { - await db().run(`CREATE TABLE "project_${projectId}" AS SELECT * FROM read_json_auto(?)`, [ - filePath, - ]) + // Determine if the file is CSV based on extension + const isCSV = filePath.toLowerCase().endsWith('.csv') + + if (isCSV) { + await db().run(`CREATE TABLE "project_${projectId}" AS SELECT * FROM read_csv_auto(?)`, [ + filePath, + ]) + } else { + await db().run(`CREATE TABLE "project_${projectId}" AS SELECT * FROM read_json_auto(?)`, [ + filePath, + ]) + } // @ts-expect-error ToDo: Fix return status(201, new Response(null)) } catch (error) { - // Check if the error is related to JSON parsing + // Check if the error is related to parsing const errorMessage = String(error) if (errorMessage.toLowerCase().includes('parse')) { + const fileType = filePath.toLowerCase().endsWith('.csv') ? 'CSV' : 'JSON' return status( 400, - ApiErrorHandler.invalidJsonErrorWithData('Invalid JSON format in uploaded file', [ - (error as Error).message, - ]), + ApiErrorHandler.invalidJsonErrorWithData( + `Invalid ${fileType} format in uploaded file`, + [(error as Error).message], + ), ) } @@ -449,7 +479,11 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) // Generate a unique temporary file name const timestamp = Date.now() const randomSuffix = Math.random().toString(36).substring(2, 8) - const tempFileName = `temp_${timestamp}_${randomSuffix}.json` + + // Determine file extension based on content type or file name + const isCSV = file.name.toLowerCase().endsWith('.csv') || file.type === 'text/csv' + const fileExtension = isCSV ? '.csv' : '.json' + const tempFileName = `temp_${timestamp}_${randomSuffix}${fileExtension}` const tempFilePath = `./temp/${tempFileName}` // Convert file to buffer and save to temporary location diff --git a/backend/src/api/wikibase/index.ts b/backend/src/api/wikibase/index.ts index 8b8da08..a603be0 100644 --- a/backend/src/api/wikibase/index.ts +++ b/backend/src/api/wikibase/index.ts @@ -27,36 +27,6 @@ export const wikibaseEntitiesApi = new Elysia({ prefix: '/api/wikibase' }) }), }) - .post( - '/:instanceId/csrf-token', - async ({ params: { instanceId }, body: { endpoint, credentials }, wikibase }) => { - const { - query: { - tokens: { csrftoken }, - }, - } = await wikibase.getCsrfToken(instanceId, endpoint, credentials) - return { - token: csrftoken, - } - }, - { - body: t.Object({ - endpoint: t.String({ - description: 'Wikibase endpoint URL', - }), - credentials: OAuthCredentials, - }), - response: t.Object({ - token: t.String(), - }), - detail: { - summary: 'Get CSRF token', - description: 'Get a CSRF token for a Wikibase instance using OAuth credentials', - tags: ['Wikibase'], - }, - }, - ) - .post( '/:instanceId/properties/fetch', async ({ params: { instanceId }, wikibase, db }) => { diff --git a/backend/tests/api/project/project.import-csv.test.ts b/backend/tests/api/project/project.import-csv.test.ts new file mode 100644 index 0000000..451ee67 --- /dev/null +++ b/backend/tests/api/project/project.import-csv.test.ts @@ -0,0 +1,76 @@ +import { projectRoutes } from '@backend/api/project' +import { UUID_REGEX_PATTERN } from '@backend/api/project/schemas' +import { closeDb, initializeDb } from '@backend/plugins/database' +import { treaty } from '@elysiajs/eden' +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' +import { Elysia } from 'elysia' + +const createTestApi = () => { + return treaty(new Elysia().use(projectRoutes)).api +} + +describe('project import with CSV', () => { + let api: ReturnType + + beforeEach(async () => { + await initializeDb(':memory:') + api = createTestApi() + }) + + afterEach(async () => { + await closeDb() + }) + + const values = [ + ['John', '30', 'New York'], + ['Jane', '25', 'Boston'], + ] + + const testcases = [ + { + name: 'with header without quotes', + csvContent: 'name,age,city\nJohn,30,New York\nJane,25,Boston', + setHasHeaders: false, + }, + { + name: 'with header with quotes', + csvContent: '"name","age","city"\n"John","30","New York"\n"Jane","25","Boston"', + setHasHeaders: false, + }, + { + name: 'without header without quotes', + csvContent: 'John,30,New York\nJane,25,Boston', + setHasHeaders: true, + hasHeaders: false, + }, + { + name: 'without header with quotes', + csvContent: '"John","30","New York"\n"Jane","25","Boston"', + setHasHeaders: true, + hasHeaders: false, + }, + ] + + test.each(testcases)('$name', async ({ csvContent, setHasHeaders, hasHeaders }) => { + const file = new File([csvContent], 'test-data.csv', { type: 'text/csv' }) + + const b = { + file, + ...(setHasHeaders ? { hasHeaders: hasHeaders } : {}), + } + + const { data, status, error } = await api.project.import.post(b) + + expect(status).toBe(201) + expect(error).toBeNull() + expect(data).toHaveProperty('data.id', expect.stringMatching(UUID_REGEX_PATTERN)) + + // @ts-expect-error Elysia Eden thinks 201 is an error + const { data: project } = await api.project({ projectId: data!.data!.id }).get() + expect(project).toHaveProperty('data') + expect(project!.data).toBeArrayOfSize(2) + + expect(project!.data[0]).toContainAnyValues(values[0]!) + expect(project!.data[1]).toContainAnyValues(values[1]!) + }) +}) diff --git a/frontend/src/features/project-management/pages/CreateProject.vue b/frontend/src/features/project-management/pages/CreateProject.vue index 75b2923..9899320 100644 --- a/frontend/src/features/project-management/pages/CreateProject.vue +++ b/frontend/src/features/project-management/pages/CreateProject.vue @@ -18,7 +18,7 @@ const { isCreating } = storeToRefs(store)

Drag and drop files to here to upload.

-

JSON files only

+

JSON or CSV files