Skip to content

feat: enable importing csv files #164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DaxServer merged 1 commit into from
Sep 27, 2025
Merged

feat: enable importing csv files #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 71 additions & 37 deletions backend/src/api/project/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' })

.post(
'/import',
async ({ db, body: { name, file }, status }) => {
async ({ db, body: { name, file, hasHeaders = true }, status }) => {
// Generate project name if not provided
const projectName = name || generateProjectName(file.name)

Expand Down Expand Up @@ -127,34 +127,40 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' })
// Save file to temporary location
const timestamp = Date.now()
const randomSuffix = Math.random().toString(36).substring(2, 8)
const tempFileName = `temp_${timestamp}_${randomSuffix}.json`

// Determine file extension based on content type or file name
const isCSV = file.name.toLowerCase().endsWith('.csv') || file.type === 'text/csv'
const fileExtension = isCSV ? '.csv' : '.json'
const tempFileName = `temp_${timestamp}_${randomSuffix}${fileExtension}`
const tempFilePath = `./temp/${tempFileName}`

const fileBuffer = await file.arrayBuffer()
const uint8Array = new Uint8Array(fileBuffer)

await Bun.write(tempFilePath, uint8Array)

// Check if JSON is parsable
try {
await Bun.file(tempFilePath).json()
} catch (parseError) {
await cleanupProject(db, project.id, tempFilePath)

const errorMessage =
parseError instanceof Error ? parseError.message : 'Failed to parse JSON'
// Extract the specific error part from the message (e.g., "Unexpected identifier 'json'")
const match = errorMessage.match(/Unexpected identifier "([^"]+)"/)
const errorDetail = match
? `JSON Parse error: Unexpected identifier "${match[1]}"`
: 'Failed to parse JSON'
// For JSON files, check if JSON is parsable
if (!isCSV) {
try {
await Bun.file(tempFilePath).json()
} catch (parseError) {
await cleanupProject(db, project.id, tempFilePath)

const errorMessage =
parseError instanceof Error ? parseError.message : 'Failed to parse JSON'
// Extract the specific error part from the message (e.g., "Unexpected identifier 'json'")
const match = errorMessage.match(/Unexpected identifier "([^"]+)"/)
const errorDetail = match
? `JSON Parse error: Unexpected identifier "${match[1]}"`
: 'Failed to parse JSON'

return status(
400,
ApiErrorHandler.invalidJsonErrorWithData('Invalid JSON format in uploaded file', [
errorDetail,
]),
)
return status(
400,
ApiErrorHandler.invalidJsonErrorWithData('Invalid JSON format in uploaded file', [
errorDetail,
]),
)
}
}

// Check if table already exists
Expand All @@ -173,14 +179,24 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' })
)
}

// First, import the JSON data into the table
await db().run(
`
CREATE TABLE "project_${project.id}" AS
SELECT * FROM read_json_auto(?)
`,
[tempFilePath],
)
// Import the data into the table based on file type
if (isCSV) {
await db().run(
`
CREATE TABLE "project_${project.id}" AS
SELECT * FROM read_csv(?, header = ?)
`,
[tempFilePath, hasHeaders],
)
} else {
await db().run(
`
CREATE TABLE "project_${project.id}" AS
SELECT * FROM read_json_auto(?)
`,
[tempFilePath],
)
}

// Check if 'id' column already exists and generate unique primary key column name
const tableInfo = await db().runAndReadAll(`PRAGMA table_info("project_${project.id}")`)
Expand Down Expand Up @@ -239,6 +255,9 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' })
error: 'Project name must be between 1 and 255 characters long if provided',
}),
),
hasHeaders: t.Optional(t.BooleanString({
default: true,
})),
}),
response: {
201: t.Object({
Expand Down Expand Up @@ -400,21 +419,32 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' })

// Try to create the table directly
try {
await db().run(`CREATE TABLE "project_${projectId}" AS SELECT * FROM read_json_auto(?)`, [
filePath,
])
// Determine if the file is CSV based on extension
const isCSV = filePath.toLowerCase().endsWith('.csv')

if (isCSV) {
await db().run(`CREATE TABLE "project_${projectId}" AS SELECT * FROM read_csv_auto(?)`, [
filePath,
])
} else {
await db().run(`CREATE TABLE "project_${projectId}" AS SELECT * FROM read_json_auto(?)`, [
filePath,
])
}

// @ts-expect-error ToDo: Fix
return status(201, new Response(null))
} catch (error) {
// Check if the error is related to JSON parsing
// Check if the error is related to parsing
const errorMessage = String(error)
if (errorMessage.toLowerCase().includes('parse')) {
const fileType = filePath.toLowerCase().endsWith('.csv') ? 'CSV' : 'JSON'
return status(
400,
ApiErrorHandler.invalidJsonErrorWithData('Invalid JSON format in uploaded file', [
(error as Error).message,
]),
ApiErrorHandler.invalidJsonErrorWithData(
`Invalid ${fileType} format in uploaded file`,
[(error as Error).message],
),
)
}

Expand Down Expand Up @@ -449,7 +479,11 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' })
// Generate a unique temporary file name
const timestamp = Date.now()
const randomSuffix = Math.random().toString(36).substring(2, 8)
const tempFileName = `temp_${timestamp}_${randomSuffix}.json`

// Determine file extension based on content type or file name
const isCSV = file.name.toLowerCase().endsWith('.csv') || file.type === 'text/csv'
const fileExtension = isCSV ? '.csv' : '.json'
const tempFileName = `temp_${timestamp}_${randomSuffix}${fileExtension}`
const tempFilePath = `./temp/${tempFileName}`

// Convert file to buffer and save to temporary location
Expand Down
30 changes: 0 additions & 30 deletions backend/src/api/wikibase/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,36 +27,6 @@ export const wikibaseEntitiesApi = new Elysia({ prefix: '/api/wikibase' })
}),
})

.post(
'/:instanceId/csrf-token',
async ({ params: { instanceId }, body: { endpoint, credentials }, wikibase }) => {
const {
query: {
tokens: { csrftoken },
},
} = await wikibase.getCsrfToken(instanceId, endpoint, credentials)
return {
token: csrftoken,
}
},
{
body: t.Object({
endpoint: t.String({
description: 'Wikibase endpoint URL',
}),
credentials: OAuthCredentials,
}),
response: t.Object({
token: t.String(),
}),
detail: {
summary: 'Get CSRF token',
description: 'Get a CSRF token for a Wikibase instance using OAuth credentials',
tags: ['Wikibase'],
},
},
)

.post(
'/:instanceId/properties/fetch',
async ({ params: { instanceId }, wikibase, db }) => {
Expand Down
76 changes: 76 additions & 0 deletions backend/tests/api/project/project.import-csv.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { projectRoutes } from '@backend/api/project'
import { UUID_REGEX_PATTERN } from '@backend/api/project/schemas'
import { closeDb, initializeDb } from '@backend/plugins/database'
import { treaty } from '@elysiajs/eden'
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
import { Elysia } from 'elysia'

const createTestApi = () => {
return treaty(new Elysia().use(projectRoutes)).api
}

describe('project import with CSV', () => {
let api: ReturnType<typeof createTestApi>

beforeEach(async () => {
await initializeDb(':memory:')
api = createTestApi()
})

afterEach(async () => {
await closeDb()
})

const values = [
['John', '30', 'New York'],
['Jane', '25', 'Boston'],
]

const testcases = [
{
name: 'with header without quotes',
csvContent: 'name,age,city\nJohn,30,New York\nJane,25,Boston',
setHasHeaders: false,
},
{
name: 'with header with quotes',
csvContent: '"name","age","city"\n"John","30","New York"\n"Jane","25","Boston"',
setHasHeaders: false,
},
{
name: 'without header without quotes',
csvContent: 'John,30,New York\nJane,25,Boston',
setHasHeaders: true,
hasHeaders: false,
},
{
name: 'without header with quotes',
csvContent: '"John","30","New York"\n"Jane","25","Boston"',
setHasHeaders: true,
hasHeaders: false,
},
]

test.each(testcases)('$name', async ({ csvContent, setHasHeaders, hasHeaders }) => {
const file = new File([csvContent], 'test-data.csv', { type: 'text/csv' })

const b = {
file,
...(setHasHeaders ? { hasHeaders: hasHeaders } : {}),
}

const { data, status, error } = await api.project.import.post(b)

expect(status).toBe(201)
expect(error).toBeNull()
expect(data).toHaveProperty('data.id', expect.stringMatching(UUID_REGEX_PATTERN))

// @ts-expect-error Elysia Eden thinks 201 is an error
const { data: project } = await api.project({ projectId: data!.data!.id }).get()
expect(project).toHaveProperty('data')
expect(project!.data).toBeArrayOfSize(2)

expect(project!.data[0]).toContainAnyValues(values[0]!)
expect(project!.data[1]).toContainAnyValues(values[1]!)
})
})
4 changes: 2 additions & 2 deletions frontend/src/features/project-management/pages/CreateProject.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ const { isCreating } = storeToRefs(store)
<label class="block text-sm font-medium text-gray-700 mb-2">Upload Data File</label>
<FileUpload
name="file"
accept=".json"
accept=".json,.csv"
custom-upload
:file-limit="1"
:preview-width="0"
Expand All @@ -33,7 +33,7 @@ const { isCreating } = storeToRefs(store)
>
<i class="pi pi-cloud-upload border-1 rounded-full p-6 !text-4xl text-info" />
<p class="mt-6 mb-0 text-gray-600">Drag and drop files to here to upload.</p>
<p class="text-xs text-gray-500 mt-2">JSON files only</p>
<p class="text-xs text-gray-500 mt-2">JSON or CSV files</p>
</div>
</template>
</FileUpload>
Expand Down