From ee357edae897fb2d84e898c100c2300fd8eeaf1a Mon Sep 17 00:00:00 2001 From: DaxServer Date: Mon, 29 Sep 2025 15:08:16 +0200 Subject: [PATCH 1/2] feat(column-operation): Implement common pattern for column operations This change introduces a common pattern for column operations in the `ColumnOperationService` class. The key changes are: 1. Adds a `executeColumnOperation` method that handles the common steps: - Get the original column type - Ensure the column is a string-like type if needed - Count the affected rows - Perform the operation if there are affected rows - Revert the column type if no rows were affected and the type was converted 2. Adds helper methods to change the column type, escape regex characters, and get the column type from the table schema. 3. Refactors the `ReplaceOperationService` to use the new `executeColumnOperation` method. 4. Adds a new `TrimWhitespaceSchema` for the trim whitespace operation API endpoint. 5. Adds tests for the new trim whitespace operation API endpoint. --- backend/src/api/project/index.ts | 68 +++++++- backend/src/api/project/schemas.ts | 8 + .../src/services/column-operation.service.ts | 118 +++++++++++++ .../src/services/replace-operation.service.ts | 117 ++----------- .../src/services/trim-whitespace.service.ts | 43 +++++ .../project/project.trim-whitespace.test.ts | 160 ++++++++++++++++++ ...lace-operation.alter-table.service.test.ts | 2 +- .../replace-operation.service.test.ts | 20 +-- .../services/trim-whitespace.service.test.ts | 119 +++++++++++++ .../components/ColumnHeaderMenu.vue | 26 ++- .../stores/project.store.ts | 12 +- 11 files changed, 571 insertions(+), 122 deletions(-) create mode 100644 backend/src/services/column-operation.service.ts create mode 100644 backend/src/services/trim-whitespace.service.ts create mode 100644 backend/tests/api/project/project.trim-whitespace.test.ts create mode 100644 backend/tests/services/trim-whitespace.service.test.ts diff --git a/backend/src/api/project/index.ts b/backend/src/api/project/index.ts index e53738d..e9ac90a 100644 --- a/backend/src/api/project/index.ts +++ b/backend/src/api/project/index.ts @@ -5,11 +5,13 @@ import { ProjectParams, ProjectResponseSchema, ReplaceOperationSchema, + TrimWhitespaceSchema, type Project, } from '@backend/api/project/schemas' import { databasePlugin } from '@backend/plugins/database' import { errorHandlerPlugin } from '@backend/plugins/error-handler' import { ReplaceOperationService } from '@backend/services/replace-operation.service' +import { TrimWhitespaceService } from '@backend/services/trim-whitespace.service' import { ApiErrorHandler } from '@backend/types/error-handler' import { ApiErrors } from '@backend/types/error-schemas' import { enhanceSchemaWithTypes, type DuckDBTablePragma } from '@backend/utils/duckdb-types' @@ -558,7 +560,7 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) const replaceService = new ReplaceOperationService(db()) try { - const affectedRows = await replaceService.performReplace({ + const affectedRows = await replaceService.performOperation({ table, column, find, @@ -584,7 +586,7 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) body: ReplaceOperationSchema, response: { 200: t.Object({ - affectedRows: t.Number(), + affectedRows: t.Integer(), }), 400: ApiErrors, 404: ApiErrors, @@ -598,3 +600,65 @@ export const projectRoutes = new Elysia({ prefix: '/api/project' }) }, }, ) + + .post( + '/:projectId/trim_whitespace', + async ({ db, params: { projectId }, body: { column }, status }) => { + const table = `project_${projectId}` + + // Check if column exists + const columnExistsReader = await db().runAndReadAll( + 'SELECT 1 FROM information_schema.columns WHERE table_name = ? AND column_name = ?', + [table, column], + ) + + if (columnExistsReader.getRows().length === 0) { + return status( + 400, + ApiErrorHandler.validationErrorWithData('Column not found', [ + `Column '${column}' does not exist in table '${table}'`, + ]), + ) + } + + const trimWhitespaceService = new TrimWhitespaceService(db()) + + try { + const affectedRows = await trimWhitespaceService.performOperation({ + table, + column, + }) + + return { + affectedRows, + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred' + return status( + 500, + ApiErrorHandler.internalServerErrorWithData( + 'Failed to perform trim whitespace operation', + [errorMessage], + ), + ) + } + }, + { + body: TrimWhitespaceSchema, + response: { + 200: t.Object({ + affectedRows: t.Integer(), + }), + 400: ApiErrors, + 404: ApiErrors, + 422: ApiErrors, + 500: ApiErrors, + }, + detail: { + summary: 'Trim leading and trailing whitespace from a column', + description: + 'Remove leading and trailing whitespace characters from all values in a specific column', + tags, + }, + }, + ) diff --git a/backend/src/api/project/schemas.ts b/backend/src/api/project/schemas.ts index 369f9d6..6c33147 100644 --- a/backend/src/api/project/schemas.ts +++ b/backend/src/api/project/schemas.ts @@ -70,3 +70,11 @@ export const ReplaceOperationSchema = t.Object({ default: false, }), }) + +// Trim whitespace operation schema +export const TrimWhitespaceSchema = t.Object({ + column: t.String({ + minLength: 1, + error: 'Column name is required and must be at least 1 character long', + }), +}) diff --git a/backend/src/services/column-operation.service.ts b/backend/src/services/column-operation.service.ts new file mode 100644 index 0000000..5afcccb --- /dev/null +++ b/backend/src/services/column-operation.service.ts @@ -0,0 +1,118 @@ +import type { DuckDBConnection, DuckDBValue } from '@duckdb/node-api' + +export interface ColumnOperationParams { + table: string + column: string +} + +export abstract class ColumnOperationService { + constructor(protected db: DuckDBConnection) {} + + /** + * Abstract method that must be implemented by subclasses to perform the specific operation + * This will be the entry point called from API endpoints + */ + public abstract performOperation(params: ColumnOperationParams): Promise + + /** + * Common pattern for column operations: + * 1. Get original column type + * 2. Ensure column is string type if needed + * 3. Count affected rows + * 4. Perform operation if rows affected + * 5. Revert column type if no rows affected and type was converted + */ + protected async executeColumnOperation( + table: string, + column: string, + operation: () => { query: string; params: DuckDBValue[] }, + countAffectedRows: () => Promise, + ): Promise { + // Get the original column type before any modifications + const originalColumnType = await this.getColumnType(table, column) + + // Check if column is string-like, if not, convert it first + const wasConverted = await this.ensureColumnIsStringType(table, column) + + // Count rows that will be affected before the update + const affectedRows = await countAffectedRows() + + // Only proceed if there are rows to update + if (affectedRows === 0) { + // Revert column type if it was converted and no rows were affected + if (wasConverted) { + await this.changeColumnType(table, column, originalColumnType) + } + return 0 + } + + // Build and execute the parameterized UPDATE query + const { query, params } = operation() + await this.db.run(query, params) + + return affectedRows + } + + /** + * Changes the column type using ALTER TABLE + */ + protected async changeColumnType(table: string, column: string, newType: string): Promise { + await this.db.run(`ALTER TABLE "${table}" ALTER "${column}" TYPE ${newType}`) + } + + /** + * Escapes special regex characters in a string + */ + protected escapeRegex(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + } + + protected async getCount(query: string, params: DuckDBValue[]): Promise { + const result = (await this.db.runAndReadAll(query, params)).getRowObjectsJson() as Array<{ + count: number + }> + + return Number(result[0]!.count) + } + + /** + * Checks if a column type is string-like (VARCHAR, TEXT, CHAR, BPCHAR) + */ + private isStringLikeType(columnType: string): boolean { + return ['VARCHAR', 'TEXT', 'CHAR', 'BPCHAR'].some((type) => columnType.includes(type)) + } + + /** + * Ensures the column is a string-like type, converting it if necessary + * Returns true if the column was converted, false otherwise + */ + private async ensureColumnIsStringType(table: string, column: string): Promise { + const columnType = await this.getColumnType(table, column) + + if (!this.isStringLikeType(columnType)) { + // Convert the column to VARCHAR + await this.changeColumnType(table, column, 'VARCHAR') + return true + } + + return false + } + + /** + * Gets the column type from the table schema + */ + private async getColumnType(table: string, column: string): Promise { + const result = await this.db.runAndReadAll(`PRAGMA table_info("${table}")`) + const columns = result.getRowObjectsJson() as Array<{ + name: string + type: string + }> + + const columnInfo = columns.find((col) => col.name === column) + if (!columnInfo) { + throw new Error(`Column '${column}' not found in table '${table}'`) + } + + return columnInfo.type.toUpperCase() + } +} diff --git a/backend/src/services/replace-operation.service.ts b/backend/src/services/replace-operation.service.ts index bb7bda2..06f0d80 100644 --- a/backend/src/services/replace-operation.service.ts +++ b/backend/src/services/replace-operation.service.ts @@ -1,54 +1,25 @@ -import type { DuckDBConnection, DuckDBValue } from '@duckdb/node-api' +import type { ColumnOperationParams } from '@backend/services/column-operation.service' +import { ColumnOperationService } from '@backend/services/column-operation.service' +import type { DuckDBValue } from '@duckdb/node-api' -export interface ReplaceOperationParams { - table: string - column: string +interface ReplaceOperationParams extends ColumnOperationParams { find: string replace: string caseSensitive: boolean wholeWord: boolean } -export class ReplaceOperationService { - constructor(private db: DuckDBConnection) {} - - /** - * Performs a replace operation on a column in a project table - */ - async performReplace(params: ReplaceOperationParams): Promise { +export class ReplaceOperationService extends ColumnOperationService { + public async performOperation(params: ReplaceOperationParams): Promise { const { table, column, find, replace, caseSensitive, wholeWord } = params - // Get the original column type before any modifications - const originalColumnType = await this.getColumnType(table, column) - - // Check if column is string-like, if not, convert it first - const wasConverted = await this.ensureColumnIsStringType(table, column) - - // Count rows that will be affected before the update - const affectedRows = await this.countAffectedRows(table, column, find, caseSensitive, wholeWord) - - // Only proceed if there are rows to update - if (affectedRows === 0) { - // Revert column type if it was converted and no rows were affected - if (wasConverted) { - await this.changeColumnType(table, column, originalColumnType) - } - return 0 - } - - // Build and execute the parameterized UPDATE query - const { query, params: queryParams } = this.buildParameterizedUpdateQuery( + return this.executeColumnOperation( table, column, - find, - replace, - caseSensitive, - wholeWord, + () => + this.buildParameterizedUpdateQuery(table, column, find, replace, caseSensitive, wholeWord), + () => this.countAffectedRows(table, column, find, caseSensitive, wholeWord), ) - - await this.db.run(query, queryParams) - - return affectedRows } /** @@ -113,7 +84,7 @@ export class ReplaceOperationService { /** * Counts the number of rows that will be affected by the replace operation */ - private async countAffectedRows( + private countAffectedRows( table: string, column: string, find: string, @@ -152,70 +123,6 @@ export class ReplaceOperationService { } } - const countBeforeReader = await this.db.runAndReadAll(query, params) - const countBeforeResult = countBeforeReader.getRowObjectsJson() - - return Number(countBeforeResult[0]?.count ?? 0) - } - - /** - * Escapes special regex characters in a string - */ - private escapeRegex(str: string): string { - return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - } - - /** - * Gets the column type from the table schema - */ - private async getColumnType(table: string, column: string): Promise { - const result = await this.db.runAndReadAll(`PRAGMA table_info("${table}")`) - const columns = result.getRowObjectsJson() as Array<{ - cid: number - name: string - type: string - pk: boolean - notnull: boolean - dflt_value: string | null - }> - - const columnInfo = columns.find((col) => col.name === column) - if (!columnInfo) { - throw new Error(`Column '${column}' not found in table '${table}'`) - } - - return columnInfo.type.toUpperCase() - } - - /** - * Checks if a column type is string-like (VARCHAR, TEXT, BLOB) - */ - private isStringLikeType(columnType: string): boolean { - const stringTypes = ['VARCHAR', 'TEXT', 'CHAR', 'BPCHAR'] - - return stringTypes.some((type) => columnType.includes(type)) - } - - /** - * Ensures the column is a string-like type, converting it if necessary - * Returns true if the column was converted, false otherwise - */ - private async ensureColumnIsStringType(table: string, column: string): Promise { - const columnType = await this.getColumnType(table, column) - - if (!this.isStringLikeType(columnType)) { - // Convert the column to VARCHAR - await this.changeColumnType(table, column, 'VARCHAR') - return true - } - - return false - } - - /** - * Changes the column type to the specified type - */ - private async changeColumnType(table: string, column: string, newType: string): Promise { - await this.db.run(`ALTER TABLE "${table}" ALTER "${column}" TYPE ${newType}`) + return this.getCount(query, params) } } diff --git a/backend/src/services/trim-whitespace.service.ts b/backend/src/services/trim-whitespace.service.ts new file mode 100644 index 0000000..7419460 --- /dev/null +++ b/backend/src/services/trim-whitespace.service.ts @@ -0,0 +1,43 @@ +import type { ColumnOperationParams } from '@backend/services/column-operation.service' +import { ColumnOperationService } from '@backend/services/column-operation.service' + +export class TrimWhitespaceService extends ColumnOperationService { + public async performOperation(params: ColumnOperationParams): Promise { + const { table, column } = params + + return this.executeColumnOperation( + table, + column, + () => this.buildParameterizedUpdateQuery(table, column), + () => this.countAffectedRows(table, column), + ) + } + + /** + * Builds a parameterized UPDATE query to safely perform trim whitespace operations + */ + private buildParameterizedUpdateQuery(table: string, column: string) { + const query = ` + UPDATE "${table}" + SET "${column}" = regexp_replace("${column}", '^\\s+|\\s+$', '', 'g') + WHERE "${column}" IS NOT NULL + AND "${column}" != regexp_replace("${column}", '^\\s+|\\s+$', '', 'g') + ` + + return { query, params: [] } + } + + /** + * Counts the number of rows that would be affected by the trim operation + */ + private countAffectedRows(table: string, column: string): Promise { + const query = ` + SELECT COUNT(*) as count + FROM "${table}" + WHERE "${column}" IS NOT NULL + AND "${column}" != regexp_replace("${column}", '^\\s+|\\s+$', '', 'g') + ` + + return this.getCount(query, []) + } +} diff --git a/backend/tests/api/project/project.trim-whitespace.test.ts b/backend/tests/api/project/project.trim-whitespace.test.ts new file mode 100644 index 0000000..e820486 --- /dev/null +++ b/backend/tests/api/project/project.trim-whitespace.test.ts @@ -0,0 +1,160 @@ +import { projectRoutes } from '@backend/api/project' +import { initializeDb } from '@backend/plugins/database' +import { treaty } from '@elysiajs/eden' +import { beforeEach, describe, expect, test } from 'bun:test' +import { Elysia } from 'elysia' +import { tmpdir } from 'node:os' + +interface TestData { + text_col: string + int_col: number + double_col: number + bool_col: boolean + date_col: string +} + +const TEST_DATA: TestData[] = [ + { + text_col: ' hello world ', + int_col: 1, + double_col: 1.5, + bool_col: true, + date_col: '2023-01-01', + }, + { text_col: '\ttab start', int_col: 2, double_col: 2.5, bool_col: false, date_col: '2023-01-02' }, + { + text_col: 'newline end\n', + int_col: 3, + double_col: 3.5, + bool_col: true, + date_col: '2023-01-03', + }, + { + text_col: ' \t multiple spaces \n ', + int_col: 4, + double_col: 4.5, + bool_col: false, + date_col: '2023-01-04', + }, + { + text_col: 'no whitespace', + int_col: 5, + double_col: 5.5, + bool_col: true, + date_col: '2023-01-05', + }, + { text_col: '', int_col: 6, double_col: 6.5, bool_col: false, date_col: '2023-01-06' }, + { text_col: ' ', int_col: 7, double_col: 7.5, bool_col: true, date_col: '2023-01-07' }, +] + +const createTestApi = () => { + return treaty(new Elysia().use(projectRoutes)).api +} +const tempFilePath = tmpdir() + '/test-trim-whitespace-data.json' + +describe('Project API - trim whitespace', () => { + let api: ReturnType + let projectId: string + + const importTestData = async () => { + await Bun.write(tempFilePath, JSON.stringify(TEST_DATA)) + + const { status, error } = await api.project({ projectId }).import.post({ + filePath: tempFilePath, + }) + + expect(error).toBeNull() + expect(status).toBe(201) + } + + beforeEach(async () => { + await initializeDb(':memory:') + api = createTestApi() + + const { data, status, error } = await api.project.post({ + name: 'Test Project for trim whitespace', + }) + expect(error).toBeNull() + expect(status).toBe(201) + projectId = (data as any)!.data!.id as string + + await importTestData() + }) + + test('should trim whitespace from VARCHAR column', async () => { + const { data, status, error } = await api.project({ projectId }).trim_whitespace.post({ + column: 'text_col', + }) + + expect(status).toBe(200) + expect(error).toBeNull() + expect(data).toEqual({ + affectedRows: 5, + }) + + // Verify the data was actually changed + const { data: projectData } = await api.project({ projectId }).get({ + query: { offset: 0, limit: 25 }, + }) + + const textValues = projectData!.data?.map((row: TestData) => row.text_col) + expect(textValues).toContain('hello world') // ' hello world ' -> 'hello world' + expect(textValues).toContain('tab start') // '\ttab start' -> 'tab start' + expect(textValues).toContain('newline end') // 'newline end\n' -> 'newline end' + expect(textValues).toContain('multiple spaces') // ' \t multiple spaces \n ' -> 'multiple spaces' + expect(textValues).toContain('no whitespace') // unchanged + expect(textValues).toContain('') // ' ' -> '' + }) + + test('should handle non-existent column', async () => { + const { data, status, error } = await api.project({ projectId }).trim_whitespace.post({ + column: 'nonexistent_column', + }) + + expect(status).toBe(400) + expect(data).toBeNull() + expect(error).toHaveProperty('status', 400) + expect(error).toHaveProperty('value', [ + { + code: 'VALIDATION', + message: 'Column not found', + details: [`Column 'nonexistent_column' does not exist in table 'project_${projectId}'`], + }, + ]) + }) + + test('should handle invalid project ID', async () => { + const invalidProjectId = 'invalid-uuid' + + const { data, status, error } = await api + .project({ projectId: invalidProjectId }) + .trim_whitespace.post({ + column: 'text_col', + }) + + expect(status).toBe(422) + expect(data).toBeNull() + expect(error).toHaveProperty('status', 422) + }) + + test('should handle missing column parameter', async () => { + // @ts-expect-error testing invalid payload + const { data, status, error } = await api.project({ projectId }).trim_whitespace.post({}) + + expect(status).toBe(422) + expect(data).toBeNull() + expect(error).toHaveProperty('status', 422) + }) + + test('should convert INTEGER column to VARCHAR and trim (0 affected rows)', async () => { + const { data, status, error } = await api.project({ projectId }).trim_whitespace.post({ + column: 'int_col', + }) + + expect(status).toBe(200) + expect(error).toBeNull() + expect(data).toEqual({ + affectedRows: 0, + }) + }) +}) diff --git a/backend/tests/services/replace-operation.alter-table.service.test.ts b/backend/tests/services/replace-operation.alter-table.service.test.ts index cc67065..be22d85 100644 --- a/backend/tests/services/replace-operation.alter-table.service.test.ts +++ b/backend/tests/services/replace-operation.alter-table.service.test.ts @@ -270,7 +270,7 @@ describe('non-string column datatype conversion', () => { expect(column!.type).toBe(initialType) // Perform replace operation - const affectedRows = await service.performReplace({ + const affectedRows = await service.performOperation({ table: 'test', column: columnName, find, diff --git a/backend/tests/services/replace-operation.service.test.ts b/backend/tests/services/replace-operation.service.test.ts index ceba8b6..31efd96 100644 --- a/backend/tests/services/replace-operation.service.test.ts +++ b/backend/tests/services/replace-operation.service.test.ts @@ -56,7 +56,7 @@ describe('ReplaceOperationService', () => { describe('performReplace', () => { test('should perform basic replace operation', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'city', find: 'New York', @@ -69,7 +69,7 @@ describe('ReplaceOperationService', () => { test('should perform case-sensitive replace operation', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'name', find: 'John', @@ -82,7 +82,7 @@ describe('ReplaceOperationService', () => { test('should perform case-insensitive replace operation', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'email', find: 'JOHN', @@ -95,7 +95,7 @@ describe('ReplaceOperationService', () => { test('should perform whole word replace operation', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'name', find: 'John', @@ -108,7 +108,7 @@ describe('ReplaceOperationService', () => { test('should handle replace with empty string', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'city', find: 'New York', @@ -127,7 +127,7 @@ describe('ReplaceOperationService', () => { ) expect( - service.performReplace({ + service.performOperation({ table, column: 'email', find: '@', @@ -146,7 +146,7 @@ describe('ReplaceOperationService', () => { ) expect( - service.performReplace({ + service.performOperation({ table, column: 'name', find: "John's", @@ -161,7 +161,7 @@ describe('ReplaceOperationService', () => { describe('edge cases and error handling', () => { test('should throw error for non-existent column', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'nonexistent_column', find: 'test', @@ -174,7 +174,7 @@ describe('ReplaceOperationService', () => { test('should handle non-existent project table', () => { expect( - service.performReplace({ + service.performOperation({ table: 'nonexistent_table', column: 'name', find: 'John', @@ -187,7 +187,7 @@ describe('ReplaceOperationService', () => { test('should handle no matching rows', () => { expect( - service.performReplace({ + service.performOperation({ table, column: 'city', find: 'NonExistentCity', diff --git a/backend/tests/services/trim-whitespace.service.test.ts b/backend/tests/services/trim-whitespace.service.test.ts new file mode 100644 index 0000000..e86b7f4 --- /dev/null +++ b/backend/tests/services/trim-whitespace.service.test.ts @@ -0,0 +1,119 @@ +import { getDb, initializeDb } from '@backend/plugins/database' +import { TrimWhitespaceService } from '@backend/services/trim-whitespace.service' +import { beforeEach, describe, expect, test } from 'bun:test' + +describe('TrimWhitespaceService', () => { + let service: TrimWhitespaceService + + beforeEach(async () => { + await initializeDb(':memory:') + const db = getDb() + service = new TrimWhitespaceService(db) + + // Create test table + await db.run(` + CREATE TABLE test ( + id INTEGER PRIMARY KEY, + text_col VARCHAR(255), + int_col INTEGER, + double_col DOUBLE, + bool_col BOOLEAN, + date_col DATE + ) + `) + + // Insert test data + await db.run(` + INSERT INTO test (id, text_col, int_col, double_col, bool_col, date_col) VALUES + (1, ' hello world ', 1, 1.5, true, '2023-01-01'), + (2, '\ttab start', 2, 2.5, false, '2023-01-02'), + (3, 'newline end\n', 3, 3.5, true, '2023-01-03'), + (4, ' \t multiple spaces \n ', 4, 4.5, false, '2023-01-04'), + (5, 'no whitespace', 5, 5.5, true, '2023-01-05'), + (6, '', 6, 6.5, false, '2023-01-06'), + (7, ' ', 7, 7.5, true, '2023-01-07'), + (8, ' \u00A0 non-breaking space ', 8, 8.5, true, '2023-01-08'), + (9, '\u2003\u2003em spaces', 9, 9.5, true, '2023-01-09'), + (10, '\u200B\u200Bzero width spaces\u200B\u200B', 10, 10.5, true, '2023-01-10'), + (11, ' mixed \t\u00A0 spaces ', 11, 11.5, true, '2023-01-11'), + (12, 'normal text', 12, 12.5, true, '2023-01-12') + `) + }) + + describe('performOperation', () => { + test('should trim whitespace from VARCHAR column', async () => { + await service.performOperation({ table: 'test', column: 'text_col' }) + + // Verify the data was actually changed + const db = getDb() + const selectResult = await db.runAndReadAll( + `SELECT text_col FROM test WHERE id < 8 ORDER BY id`, + ) + const rows = selectResult.getRowObjectsJson() + + expect(rows[0]!.text_col).toBe('hello world') // ' hello world ' -> 'hello world' + expect(rows[1]!.text_col).toBe('tab start') // '\ttab start' -> 'tab start' (tab trimmed) + expect(rows[2]!.text_col).toBe('newline end') // 'newline end\n' -> 'newline end' (newline trimmed) + expect(rows[3]!.text_col).toBe('multiple spaces') // ' \t multiple spaces \n ' -> 'multiple spaces' + expect(rows[4]!.text_col).toBe('no whitespace') // unchanged + expect(rows[5]!.text_col).toBe('') // unchanged + expect(rows[6]!.text_col).toBe('') // ' ' -> '' (affected) + }) + + test('should handle non-existent column', () => { + expect( + service.performOperation({ table: 'test', column: 'nonexistent_col' }), + ).rejects.toThrowError(`Column 'nonexistent_col' not found in table 'test'`) + }) + + test('should handle non-existent table', () => { + expect( + service.performOperation({ table: 'nonexistent_table', column: 'text_col' }), + ).rejects.toThrowError(/Table.*nonexistent_table.*does not exist/) + }) + + test('should handle empty string values', async () => { + await service.performOperation({ table: 'test', column: 'text_col' }) + + const db = getDb() + const selectResult = await db.runAndReadAll( + 'SELECT text_col FROM test WHERE id BETWEEN 6 AND 7 ORDER BY id', + ) + const rows = selectResult.getRowObjectsJson() + + expect(rows[0]!.text_col).toBe('') // unchanged + expect(rows[1]!.text_col).toBe('') // ' ' -> '' (affected) + }) + + test('should handle INTEGER column (0 affected rows)', () => { + expect(service.performOperation({ table: 'test', column: 'int_col' })).resolves.toBe(0) + }) + + test('should handle DOUBLE column (0 affected rows)', () => { + expect(service.performOperation({ table: 'test', column: 'double_col' })).resolves.toBe(0) + }) + + test('should handle BOOLEAN column (0 affected rows)', () => { + expect(service.performOperation({ table: 'test', column: 'bool_col' })).resolves.toBe(0) + }) + + test('should handle DATE column (0 affected rows)', () => { + expect(service.performOperation({ table: 'test', column: 'date_col' })).resolves.toBe(0) + }) + + test('should handle special characters and unicode', async () => { + await service.performOperation({ table: 'test', column: 'text_col' }) + + const db = getDb() + const selectResult = await db.runAndReadAll( + 'SELECT text_col FROM test WHERE id BETWEEN 8 AND 11 ORDER BY id', + ) + const rows = selectResult.getRowObjectsJson() + + expect(rows[0]!.text_col).toBe('\u00A0 non-breaking space') // ' \u00A0 non-breaking space ' -> '\u00A0 non-breaking space' + expect(rows[1]!.text_col).toBe('\u2003\u2003em spaces') // unchanged (leading em spaces) + expect(rows[2]!.text_col).toBe('\u200B\u200Bzero width spaces\u200B\u200B') // unchanged (zero width spaces) + expect(rows[3]!.text_col).toBe('mixed \t\u00A0 spaces') // ' mixed \t\u00A0 spaces ' -> 'mixed \t\u00A0 spaces' + }) + }) +}) diff --git a/frontend/src/features/data-processing/components/ColumnHeaderMenu.vue b/frontend/src/features/data-processing/components/ColumnHeaderMenu.vue index 5973a4b..f117500 100644 --- a/frontend/src/features/data-processing/components/ColumnHeaderMenu.vue +++ b/frontend/src/features/data-processing/components/ColumnHeaderMenu.vue @@ -7,11 +7,13 @@ const props = defineProps<{ const emit = defineEmits(['replaceCompleted']) -const { showSuccess, showWarning } = useErrorHandling() +const { showSuccess, showWarning, showError } = useErrorHandling() +const api = useApi() const menu = ref() const isOpen = ref(false) const showReplaceDialog = ref(false) +const projectId = useRouteParams('id') as Ref const handleReplaceCompleted = (affectedRows: number) => { if (affectedRows === 0) { @@ -22,6 +24,26 @@ const handleReplaceCompleted = (affectedRows: number) => { } } +const handleTrimWhitespace = async () => { + const { data, error } = await api.project({ projectId: projectId.value }).trim_whitespace.post({ + column: props.columnField, + }) + + if (error?.value) { + showError(error.value as ExtendedError[]) + return + } + + const affectedRows = data?.affectedRows || 0 + + if (affectedRows === 0) { + showWarning('Trim whitespace completed: No rows were affected') + } else { + showSuccess(`Trim whitespace completed: ${affectedRows} rows affected`) + emit('replaceCompleted') + } +} + const menuItems = ref([ { label: 'Sort', @@ -65,7 +87,7 @@ const menuItems = ref([ items: [ { label: 'Trim leading and trailing whitespace', - command: () => console.log(`Trim whitespace in ${props.columnHeader}`), + command: handleTrimWhitespace, }, { label: 'Collapse consecutive whitespace', diff --git a/frontend/src/features/project-management/stores/project.store.ts b/frontend/src/features/project-management/stores/project.store.ts index eda5cbe..a377593 100644 --- a/frontend/src/features/project-management/stores/project.store.ts +++ b/frontend/src/features/project-management/stores/project.store.ts @@ -5,6 +5,8 @@ export const useProjectStore = defineStore('project', () => { const { showError } = useErrorHandling() const { generateColumns } = useColumnGeneration() + const projectId = ref() + // State const data = ref([]) const meta = ref({ @@ -22,14 +24,16 @@ export const useProjectStore = defineStore('project', () => { const currentLimit = ref(25) // Actions - const fetchProject = async (projectId: string, offset = 0, limit = 25) => { + const fetchProject = async (_projectId: string, offset = 0, limit = 25) => { isLoading.value = true // Store current pagination state currentOffset.value = offset currentLimit.value = limit - const { data: rows, error } = await api.project({ projectId }).get({ query: { offset, limit } }) + const { data: rows, error } = await api + .project({ projectId: _projectId }) + .get({ query: { offset, limit } }) if (error) { showError(error.value) @@ -37,6 +41,9 @@ export const useProjectStore = defineStore('project', () => { return } + // Store projectId + projectId.value = _projectId as UUID + data.value = rows.data meta.value = rows.meta columns.value = generateColumns(rows.meta.schema) @@ -87,6 +94,7 @@ export const useProjectStore = defineStore('project', () => { // State data, meta, + projectId, isLoading, columns, columnsForSchema, From f67ebd4b328f190968f3f867a30a1062178a488e Mon Sep 17 00:00:00 2001 From: DaxServer Date: Mon, 29 Sep 2025 15:31:41 +0200 Subject: [PATCH 2/2] feat: use transactions --- .../src/services/column-operation.service.ts | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/backend/src/services/column-operation.service.ts b/backend/src/services/column-operation.service.ts index 5afcccb..c58133b 100644 --- a/backend/src/services/column-operation.service.ts +++ b/backend/src/services/column-operation.service.ts @@ -10,17 +10,16 @@ export abstract class ColumnOperationService { /** * Abstract method that must be implemented by subclasses to perform the specific operation - * This will be the entry point called from API endpoints + * This will be the entry point called from the API endpoints */ public abstract performOperation(params: ColumnOperationParams): Promise /** * Common pattern for column operations: - * 1. Get original column type - * 2. Ensure column is string type if needed - * 3. Count affected rows - * 4. Perform operation if rows affected - * 5. Revert column type if no rows affected and type was converted + * 1. Ensure column is string type if needed + * 2. Count affected rows before operation + * 3. Perform operation if rows affected + * 4. Rollback if no rows affected or operation failed */ protected async executeColumnOperation( table: string, @@ -28,29 +27,32 @@ export abstract class ColumnOperationService { operation: () => { query: string; params: DuckDBValue[] }, countAffectedRows: () => Promise, ): Promise { - // Get the original column type before any modifications - const originalColumnType = await this.getColumnType(table, column) + await this.db.run('BEGIN TRANSACTION') - // Check if column is string-like, if not, convert it first - const wasConverted = await this.ensureColumnIsStringType(table, column) + try { + // Check if column is string-like, if not, convert it first + await this.ensureColumnIsStringType(table, column) - // Count rows that will be affected before the update - const affectedRows = await countAffectedRows() + // Count rows that will be affected before the update + const affectedRows = await countAffectedRows() - // Only proceed if there are rows to update - if (affectedRows === 0) { - // Revert column type if it was converted and no rows were affected - if (wasConverted) { - await this.changeColumnType(table, column, originalColumnType) + // Only proceed if there are rows to update + if (affectedRows === 0) { + await this.db.run('ROLLBACK') + + return affectedRows } - return 0 - } - // Build and execute the parameterized UPDATE query - const { query, params } = operation() - await this.db.run(query, params) + // Build and execute the parameterized UPDATE query + const { query, params } = operation() + await this.db.run(query, params) + await this.db.run('COMMIT') - return affectedRows + return affectedRows + } catch (error) { + await this.db.run('ROLLBACK') + throw error + } } /** @@ -84,18 +86,14 @@ export abstract class ColumnOperationService { /** * Ensures the column is a string-like type, converting it if necessary - * Returns true if the column was converted, false otherwise */ - private async ensureColumnIsStringType(table: string, column: string): Promise { + private async ensureColumnIsStringType(table: string, column: string): Promise { const columnType = await this.getColumnType(table, column) if (!this.isStringLikeType(columnType)) { // Convert the column to VARCHAR await this.changeColumnType(table, column, 'VARCHAR') - return true } - - return false } /**