From c03b2c4501689ead25510dea7d9e39bd258df557 Mon Sep 17 00:00:00 2001 From: John Doe Date: Thu, 8 Jan 2026 20:11:24 +0100 Subject: [PATCH 1/7] refactor: add general file sink logic --- .../utils/src/lib/file-sink-json.int.test.ts | 138 ++++++++ packages/utils/src/lib/file-sink-json.ts | 60 ++++ .../utils/src/lib/file-sink-json.unit.test.ts | 216 +++++++++++++ .../utils/src/lib/file-sink-text.int.test.ts | 184 +++++++++++ packages/utils/src/lib/file-sink-text.ts | 147 +++++++++ .../utils/src/lib/file-sink-text.unit.test.ts | 295 ++++++++++++++++++ packages/utils/src/lib/sink-source.types.ts | 48 +++ 7 files changed, 1088 insertions(+) create mode 100644 packages/utils/src/lib/file-sink-json.int.test.ts create mode 100644 packages/utils/src/lib/file-sink-json.ts create mode 100644 packages/utils/src/lib/file-sink-json.unit.test.ts create mode 100644 packages/utils/src/lib/file-sink-text.int.test.ts create mode 100644 packages/utils/src/lib/file-sink-text.ts create mode 100644 packages/utils/src/lib/file-sink-text.unit.test.ts create mode 100644 packages/utils/src/lib/sink-source.types.ts diff --git a/packages/utils/src/lib/file-sink-json.int.test.ts b/packages/utils/src/lib/file-sink-json.int.test.ts new file mode 100644 index 000000000..c331d8d0a --- /dev/null +++ b/packages/utils/src/lib/file-sink-json.int.test.ts @@ -0,0 +1,138 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { teardownTestFolder } from '@code-pushup/test-utils'; +import { JsonlFileSink, recoverJsonlFile } from './file-sink-json.js'; + +describe('JsonlFileSink integration', () => { + const baseDir = path.join(os.tmpdir(), 'file-sink-json-int-tests'); + const testFile = path.join(baseDir, 'test-data.jsonl'); + + beforeAll(async () => { + await fs.promises.mkdir(baseDir, { recursive: true }); + }); + + beforeEach(async () => { + try { + await fs.promises.unlink(testFile); + } catch { + // File doesn't exist, which is fine + } + }); + + afterAll(async () => { + await teardownTestFolder(baseDir); + }); + + describe('file operations', () => { + const testData = [ + { id: 1, name: 'Alice', active: true }, + { id: 2, name: 'Bob', active: false }, + { id: 3, name: 'Charlie', active: true }, + ]; + + it('should write and read JSONL files', async () => { + const sink = new JsonlFileSink({ filePath: testFile }); + + // Open and write data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + expect(fs.existsSync(testFile)).toBe(true); + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent.trim().split('\n'); + expect(lines).toStrictEqual([ + '{"id":1,"name":"Alice","active":true}', + '{"id":2,"name":"Bob","active":false}', + '{"id":3,"name":"Charlie","active":true}', + ]); + + lines.forEach((line, index) => { + const parsed = JSON.parse(line); + expect(parsed).toStrictEqual(testData[index]); + }); + }); + + it('should recover data from JSONL files', async () => { + const jsonlContent = `${testData.map(item => JSON.stringify(item)).join('\n')}\n`; + fs.writeFileSync(testFile, jsonlContent); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: testData, + errors: [], + partialTail: null, + }); + }); + + it('should handle JSONL files with parse errors', async () => { + const mixedContent = + '{"id":1,"name":"Alice"}\n' + + 'invalid json line\n' + + '{"id":2,"name":"Bob"}\n' + + '{"id":3,"name":"Charlie","incomplete":\n'; + + fs.writeFileSync(testFile, mixedContent); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: [ + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, + ], + errors: [ + expect.objectContaining({ line: 'invalid json line' }), + expect.objectContaining({ + line: '{"id":3,"name":"Charlie","incomplete":', + }), + ], + partialTail: '{"id":3,"name":"Charlie","incomplete":', + }); + }); + + it('should recover data using JsonlFileSink.recover()', async () => { + const sink = new JsonlFileSink({ filePath: testFile }); + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + expect(sink.recover()).toStrictEqual({ + records: testData, + errors: [], + partialTail: null, + }); + }); + + describe('edge cases', () => { + it('should handle empty files', async () => { + fs.writeFileSync(testFile, ''); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle files with only whitespace', async () => { + fs.writeFileSync(testFile, ' \n \n\t\n'); + + expect(recoverJsonlFile(testFile)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle non-existent files', async () => { + const nonExistentFile = path.join(baseDir, 'does-not-exist.jsonl'); + + expect(recoverJsonlFile(nonExistentFile)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + }); + }); +}); diff --git a/packages/utils/src/lib/file-sink-json.ts b/packages/utils/src/lib/file-sink-json.ts new file mode 100644 index 000000000..646cd82b1 --- /dev/null +++ b/packages/utils/src/lib/file-sink-json.ts @@ -0,0 +1,60 @@ +import * as fs from 'node:fs'; +import { + type FileOutput, + FileSink, + type FileSinkOptions, + stringDecode, + stringEncode, + stringRecover, +} from './file-sink-text.js'; +import type { RecoverOptions, RecoverResult } from './sink-source.types.js'; + +export const jsonlEncode = < + T extends Record = Record, +>( + input: T, +): FileOutput => JSON.stringify(input); + +export const jsonlDecode = < + T extends Record = Record, +>( + output: FileOutput, +): T => JSON.parse(stringDecode(output)) as T; + +export function recoverJsonlFile< + T extends Record = Record, +>(filePath: string, opts: RecoverOptions = {}): RecoverResult { + return stringRecover(filePath, jsonlDecode, opts); +} + +export class JsonlFileSink< + T extends Record = Record, +> extends FileSink { + constructor(options: FileSinkOptions) { + const { filePath, ...fileOptions } = options; + super({ + ...fileOptions, + filePath, + recover: () => recoverJsonlFile(filePath), + finalize: () => { + // No additional finalization needed for JSONL files + }, + }); + } + + override encode(input: T): FileOutput { + return stringEncode(jsonlEncode(input)); + } + + override decode(output: FileOutput): T { + return jsonlDecode(stringDecode(output)); + } + + override repack(outputPath?: string): void { + const { records } = this.recover(); + fs.writeFileSync( + outputPath ?? this.getFilePath(), + records.map(this.encode).join(''), + ); + } +} diff --git a/packages/utils/src/lib/file-sink-json.unit.test.ts b/packages/utils/src/lib/file-sink-json.unit.test.ts new file mode 100644 index 000000000..a920c8cbe --- /dev/null +++ b/packages/utils/src/lib/file-sink-json.unit.test.ts @@ -0,0 +1,216 @@ +import { vol } from 'memfs'; +import * as fs from 'node:fs'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { + JsonlFileSink, + jsonlDecode, + jsonlEncode, + recoverJsonlFile, +} from './file-sink-json.js'; + +describe('jsonlEncode', () => { + it('should encode object to JSON string', () => { + const obj = { key: 'value', number: 42 }; + expect(jsonlEncode(obj)).toBe(JSON.stringify(obj)); + }); + + it('should handle nested objects', () => { + const obj = { nested: { deep: 'value' }, array: [1, 2, 3] }; + expect(jsonlEncode(obj)).toBe(JSON.stringify(obj)); + }); + + it('should handle empty object', () => { + expect(jsonlEncode({})).toBe('{}'); + }); +}); + +describe('jsonlDecode', () => { + it('should decode JSON string to object', () => { + const obj = { key: 'value', number: 42 }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(jsonlDecode(jsonStr)).toStrictEqual(obj); + }); + + it('should handle nested objects', () => { + const obj = { nested: { deep: 'value' }, array: [1, 2, 3] }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(jsonlDecode(jsonStr)).toStrictEqual(obj); + }); + + it('should trim whitespace before parsing', () => { + const obj = { key: 'value' }; + const jsonStr = ` ${JSON.stringify(obj)} \n`; + expect(jsonlDecode(jsonStr)).toStrictEqual(obj); + }); + + it('should throw on invalid JSON', () => { + expect(() => jsonlDecode('invalid json\n')).toThrow('Unexpected token'); + }); + + it('should handle Buffer input', () => { + const obj = { key: 'value', number: 42 }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(jsonlDecode(Buffer.from(jsonStr))).toStrictEqual(obj); + }); + + it('should handle primitive JSON values', () => { + expect(jsonlDecode('"string"\n')).toBe('string'); + expect(jsonlDecode('42\n')).toBe(42); + expect(jsonlDecode('true\n')).toBe(true); + expect(jsonlDecode('null\n')).toBeNull(); + }); +}); + +describe('recoverJsonlFile', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + it('should recover JSONL file with single object', () => { + const filePath = '/tmp/recover-single.jsonl'; + const obj = { key: 'value', number: 42 }; + fs.writeFileSync(filePath, `${JSON.stringify(obj)}\n`); + + expect(recoverJsonlFile(filePath)).toStrictEqual({ + records: [obj], + errors: [], + partialTail: null, + }); + }); + + it('should recover JSONL file with multiple objects', () => { + const filePath = '/tmp/recover-multi.jsonl'; + const obj1 = { id: 1, name: 'first' }; + const obj2 = { id: 2, name: 'second' }; + fs.writeFileSync( + filePath, + `${JSON.stringify(obj1)}\n${JSON.stringify(obj2)}\n`, + ); + + expect(recoverJsonlFile(filePath)).toStrictEqual({ + records: [obj1, obj2], + errors: [], + partialTail: null, + }); + }); + + it('should handle JSON parsing errors', () => { + const filePath = '/tmp/recover-error.jsonl'; + fs.writeFileSync( + filePath, + '{"valid": "json"}\ninvalid json line\n{"id":3,"name":"Charlie","incomplete":\n', + ); + + const result = recoverJsonlFile(filePath); + expect(result.records).toStrictEqual([{ valid: 'json' }]); + expect(result.errors).toStrictEqual([ + expect.objectContaining({ line: 'invalid json line' }), + expect.objectContaining({ + line: '{"id":3,"name":"Charlie","incomplete":', + }), + ]); + expect(result.partialTail).toBe('{"id":3,"name":"Charlie","incomplete":'); + }); + + it('should support keepInvalid option', () => { + const filePath = '/tmp/recover-keep-invalid.jsonl'; + fs.writeFileSync(filePath, '{"valid": "json"}\ninvalid json\n'); + + const result = recoverJsonlFile(filePath, { keepInvalid: true }); + expect(result.records).toStrictEqual([ + { valid: 'json' }, + { __invalid: true, lineNo: 2, line: 'invalid json' }, + ]); + expect(result.errors).toHaveLength(1); + }); + + it('should handle empty files', () => { + const filePath = '/tmp/recover-empty.jsonl'; + fs.writeFileSync(filePath, ''); + + expect(recoverJsonlFile(filePath)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle file read errors gracefully', () => { + expect(recoverJsonlFile('/nonexistent/file.jsonl')).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); +}); + +describe('JsonlFileSink', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + type JsonObj = { key: string; number: number }; + + it('should encode objects as JSON', () => { + const sink = new JsonlFileSink({ + filePath: '/tmp/jsonl-test.jsonl', + }); + const obj = { key: 'value', number: 42 }; + expect(sink.encode(obj)).toBe(`${JSON.stringify(obj)}\n`); + }); + + it('should decode JSON strings to objects', () => { + const sink = new JsonlFileSink({ + filePath: '/tmp/jsonl-test.jsonl', + }); + const obj = { key: 'value', number: 42 }; + const jsonStr = `${JSON.stringify(obj)}\n`; + expect(sink.decode(jsonStr)).toStrictEqual(obj); + }); + + it('should handle file operations with JSONL format', () => { + const filePath = '/tmp/jsonl-file-ops-test.jsonl'; + const sink = new JsonlFileSink({ filePath }); + sink.open(); + + const obj1 = { key: 'value', number: 42 }; + const obj2 = { key: 'value', number: 42 }; + sink.write(obj1); + sink.write(obj2); + sink.close(); + + const recovered = sink.recover(); + expect(recovered.records).toStrictEqual([obj1, obj2]); + }); + + it('repack() should recover records and write them to output path', () => { + const filePath = '/tmp/jsonl-repack-test.jsonl'; + const sink = new JsonlFileSink({ filePath }); + const records = [ + { key: 'value', number: 42 }, + { key: 'value', number: 42 }, + ]; + + fs.writeFileSync( + filePath, + `${records.map(record => JSON.stringify(record)).join('\n')}\n`, + ); + + const outputPath = '/tmp/jsonl-repack-output.jsonl'; + sink.repack(outputPath); + expect(fs.readFileSync(outputPath, 'utf8')).toBe( + `${JSON.stringify(records[0])}\n${JSON.stringify(records[1])}\n`, + ); + }); +}); diff --git a/packages/utils/src/lib/file-sink-text.int.test.ts b/packages/utils/src/lib/file-sink-text.int.test.ts new file mode 100644 index 000000000..19ea34fb0 --- /dev/null +++ b/packages/utils/src/lib/file-sink-text.int.test.ts @@ -0,0 +1,184 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { teardownTestFolder } from '@code-pushup/test-utils'; +import { FileSink, stringRecover } from './file-sink-text.js'; + +describe('FileSink integration', () => { + const baseDir = path.join(os.tmpdir(), 'file-sink-text-int-tests'); + const testFile = path.join(baseDir, 'test-data.txt'); + + beforeAll(async () => { + await fs.promises.mkdir(baseDir, { recursive: true }); + }); + + beforeEach(async () => { + try { + await fs.promises.unlink(testFile); + } catch { + // File doesn't exist, which is fine + } + }); + + afterAll(async () => { + await teardownTestFolder(baseDir); + }); + + describe('file operations', () => { + const testData = ['line1', 'line2', 'line3']; + + it('should write and read text files', async () => { + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + }); + + // Open and write data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + expect(fs.existsSync(testFile)).toBe(true); + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent.trim().split('\n'); + expect(lines).toStrictEqual(testData); + + lines.forEach((line, index) => { + expect(line).toStrictEqual(testData[index]); + }); + }); + + it('should recover data from text files', async () => { + const content = `${testData.join('\n')}\n`; + fs.writeFileSync(testFile, content); + + expect(stringRecover(testFile, (line: string) => line)).toStrictEqual({ + records: testData, + errors: [], + partialTail: null, + }); + }); + + it('should handle text files with parse errors', async () => { + const mixedContent = 'valid\ninvalid\nanother\n'; + fs.writeFileSync(testFile, mixedContent); + + expect( + stringRecover(testFile, (line: string) => { + if (line === 'invalid') throw new Error('Invalid line'); + return line.toUpperCase(); + }), + ).toStrictEqual({ + records: ['VALID', 'ANOTHER'], + errors: [ + expect.objectContaining({ + lineNo: 2, + line: 'invalid', + error: expect.any(Error), + }), + ], + partialTail: 'invalid', + }); + }); + + it('should repack file with recovered data', async () => { + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + }); + + // Write initial data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + // Repack to the same file + sink.repack(); + + // Verify the content is still correct + const fileContent = fs.readFileSync(testFile, 'utf8'); + const lines = fileContent + .trim() + .split('\n') + .filter(line => line.length > 0); + expect(lines).toStrictEqual(testData); + }); + + it('should repack file to different output path', async () => { + const outputPath = path.join(baseDir, 'repacked.txt'); + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + }); + + // Write initial data + sink.open(); + testData.forEach(item => sink.write(item)); + sink.close(); + + // Repack to different file + sink.repack(outputPath); + + // Verify the original file is unchanged + expect(fs.existsSync(testFile)).toBe(true); + + // Verify the repacked file has correct content + expect(fs.existsSync(outputPath)).toBe(true); + const fileContent = fs.readFileSync(outputPath, 'utf8'); + const lines = fileContent + .trim() + .split('\n') + .filter(line => line.length > 0); + expect(lines).toStrictEqual(testData); + }); + + it('should call finalize function when provided', async () => { + let finalized = false; + const sink = new FileSink({ + filePath: testFile, + recover: () => stringRecover(testFile, (line: string) => line), + finalize: () => { + finalized = true; + }, + }); + + sink.finalize(); + expect(finalized).toBe(true); + }); + }); + + describe('edge cases', () => { + it('should handle empty files', async () => { + fs.writeFileSync(testFile, ''); + + expect(stringRecover(testFile, (line: string) => line)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle files with only whitespace', async () => { + fs.writeFileSync(testFile, ' \n \n\t\n'); + + expect(stringRecover(testFile, (line: string) => line)).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + + it('should handle non-existent files', async () => { + const nonExistentFile = path.join(baseDir, 'does-not-exist.txt'); + + expect( + stringRecover(nonExistentFile, (line: string) => line), + ).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); + }); +}); diff --git a/packages/utils/src/lib/file-sink-text.ts b/packages/utils/src/lib/file-sink-text.ts new file mode 100644 index 000000000..3cafacbe4 --- /dev/null +++ b/packages/utils/src/lib/file-sink-text.ts @@ -0,0 +1,147 @@ +import * as fs from 'node:fs'; +import { existsSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; +import type { + RecoverOptions, + RecoverResult, + Recoverable, + Sink, +} from './sink-source.types.js'; + +export const stringDecode = (output: O): I => { + const str = Buffer.isBuffer(output) + ? output.toString('utf8') + : String(output); + return str as unknown as I; +}; + +export const stringEncode = (input: I): O => + `${typeof input === 'string' ? input : JSON.stringify(input)}\n` as O; + +export const stringRecover = function ( + filePath: string, + decode: (output: O) => I, + opts: RecoverOptions = {}, +): RecoverResult { + const records: I[] = []; + const errors: { lineNo: number; line: string; error: Error }[] = []; + let partialTail: string | null = null; + + try { + const content = fs.readFileSync(filePath, 'utf8'); + const lines = content.trim().split('\n'); + let lineNo = 0; + + for (const line of lines) { + lineNo++; + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const record = decode(trimmedLine as O); + records.push(record); + } catch (error) { + const info = { lineNo, line, error: error as Error }; + errors.push(info); + + if (opts.keepInvalid) { + records.push({ __invalid: true, lineNo, line } as any); + } + + partialTail = line; + } + } + } catch { + return { records: [], errors: [], partialTail: null }; + } + + return { records, errors, partialTail }; +}; + +export type FileSinkOptions = { + filePath: string; + recover?: () => RecoverResult; + finalize?: () => void; +}; + +export type FileInput = Buffer | string; +export type FileOutput = Buffer | string; + +export class FileSink + implements Sink, Recoverable +{ + #fd: number | null = null; + options: FileSinkOptions; + + constructor(options: FileSinkOptions) { + this.options = options; + } + + isClosed(): boolean { + return this.#fd == null; + } + + encode(input: I): O { + return stringEncode(input as any); + } + + decode(output: O): I { + return stringDecode(output as any); + } + getFilePath(): string { + return this.options.filePath; + } + + open(withRepack: boolean = false): void { + const dir = path.dirname(this.options.filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + if (withRepack) { + this.repack(this.options.filePath); + } + this.#fd = fs.openSync(this.options.filePath, 'a'); + } + + close(): void { + if (this.#fd == null) { + return; + } + fs.closeSync(this.#fd); + this.#fd = null; + } + + write(input: I): void { + if (this.#fd == null) { + return; + } // Silently ignore if not open + const encoded = this.encode(input); + try { + fs.writeSync(this.#fd, encoded as any); + } catch { + // Silently ignore write errors (e.g., EBADF in test environments with mocked fs) + } + } + + recover(): RecoverResult { + const dir = path.dirname(this.options.filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + return this.options.recover!() as RecoverResult; + } + + repack(outputPath?: string): void { + const { records } = this.recover(); + fs.writeFileSync( + outputPath ?? this.getFilePath(), + records.map(this.encode).join('\n'), + ); + } + + finalize(): void { + this.options.finalize!(); + } +} diff --git a/packages/utils/src/lib/file-sink-text.unit.test.ts b/packages/utils/src/lib/file-sink-text.unit.test.ts new file mode 100644 index 000000000..f76cf13d4 --- /dev/null +++ b/packages/utils/src/lib/file-sink-text.unit.test.ts @@ -0,0 +1,295 @@ +import { vol } from 'memfs'; +import * as fs from 'node:fs'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { + FileSink, + type FileSinkOptions, + stringDecode, + stringEncode, + stringRecover, +} from './file-sink-text.js'; + +describe('stringEncode', () => { + it('stringEncode() should encode string input with newline', () => { + const str = 'test string'; + expect(stringEncode(str)).toBe(`${str}\n`); + }); + + it('stringEncode() should encode non-string input as JSON with newline', () => { + const obj = { key: 'value', number: 42 }; + expect(stringEncode(obj)).toBe(`${JSON.stringify(obj)}\n`); + }); + + it('stringEncode() should handle null input', () => { + expect(stringEncode(null)).toBe('null\n'); + }); + + it('stringEncode() should handle undefined input', () => { + expect(stringEncode(undefined)).toBe('undefined\n'); + }); +}); + +describe('stringDecode', () => { + it('stringDecode() should decode Buffer to string', () => { + const str = 'test content'; + expect(stringDecode(Buffer.from(str))).toBe(str); + }); + + it('stringDecode() should return string input as-is', () => { + const str = 'test string'; + expect(stringDecode(str)).toBe(str); + }); +}); + +describe('stringRecover', () => { + it('stringRecover() should recover records from valid file content', () => { + const filePath = '/tmp/stringRecover-test.txt'; + vol.fromJSON({ + [filePath]: 'line1\nline2\nline3\n', + }); + + expect(stringRecover(filePath, (line: string) => line)).toStrictEqual({ + records: ['line1', 'line2', 'line3'], + errors: [], + partialTail: null, + }); + }); + + it('stringRecover() should recover records and apply decode function', () => { + const filePath = '/tmp/stringRecover-test.txt'; + vol.fromJSON({ + [filePath]: 'line1\nline2\nline3\n', + }); + + expect( + stringRecover(filePath, (line: string) => line.toUpperCase()), + ).toStrictEqual({ + records: ['LINE1', 'LINE2', 'LINE3'], + errors: [], + partialTail: null, + }); + }); + + it('stringRecover() should skip empty lines', () => { + const filePath = '/tmp/stringRecover-empty-test.txt'; + vol.fromJSON({ + [filePath]: 'line1\n\nline2\n', + }); + + expect(stringRecover(filePath, (line: string) => line)).toStrictEqual({ + records: ['line1', 'line2'], + errors: [], + partialTail: null, + }); + }); + + it('stringRecover() should handle decode errors and continue processing', () => { + const filePath = '/tmp/stringRecover-error-test.txt'; + vol.fromJSON({ + [filePath]: 'valid\ninvalid\nanother', + }); + + expect( + stringRecover(filePath, (line: string) => { + if (line === 'invalid') throw new Error('Invalid line'); + return line.toUpperCase(); + }), + ).toStrictEqual({ + records: ['VALID', 'ANOTHER'], + errors: [ + { + lineNo: 2, + line: 'invalid', + error: expect.any(Error), + }, + ], + partialTail: 'invalid', + }); + }); + + it('stringRecover() should include invalid records when keepInvalid option is true', () => { + const filePath = '/tmp/stringRecover-invalid-test.txt'; + vol.fromJSON({ + [filePath]: 'valid\ninvalid\n', + }); + + expect( + stringRecover( + filePath, + (line: string) => { + if (line === 'invalid') throw new Error('Invalid line'); + return line.toUpperCase(); + }, + { keepInvalid: true }, + ), + ).toStrictEqual({ + records: ['VALID', { __invalid: true, lineNo: 2, line: 'invalid' }], + errors: [expect.any(Object)], + partialTail: 'invalid', + }); + }); + + it('stringRecover() should handle file read errors gracefully', () => { + expect( + stringRecover('/nonexistent/file.txt', (line: string) => line), + ).toStrictEqual({ + records: [], + errors: [], + partialTail: null, + }); + }); +}); + +describe('FileSink', () => { + it('constructor should create instance with options', () => { + const options: FileSinkOptions = { + filePath: '/tmp/test-file.txt', + recover: vi + .fn() + .mockReturnValue({ records: [], errors: [], partialTail: null }), + finalize: vi.fn(), + }; + expect(new FileSink(options).options).toBe(options); + }); + + it('getFilePath() should return the file path', () => { + const filePath = '/tmp/test-file.txt'; + const sink = new FileSink({ filePath }); + expect(sink.getFilePath()).toBe(filePath); + }); + + it('encode() should encode input using stringEncode', () => { + const sink = new FileSink({ filePath: '/tmp/test.txt' }); + const str = 'test input'; + expect(sink.encode(str)).toBe(`${str}\n`); + }); + + it('decode() should decode output using stringDecode', () => { + const sink = new FileSink({ filePath: '/tmp/test.txt' }); + const str = 'test output'; + expect(sink.decode(str)).toBe(str); + }); + + it('open() should handle directory creation and file opening', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(fs.existsSync('/tmp/test-file.txt')).toBe(true); + }); + + it('open() should repack file when withRepack is true', () => { + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + recover: vi + .fn() + .mockReturnValue({ records: [], errors: [], partialTail: null }), + }); + const spy = vi.spyOn(sink, 'repack'); + sink.open(true); + expect(spy).toHaveBeenCalledWith('/tmp/test-file.txt'); + }); + + it('close() should close file descriptor if open', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(() => sink.close()).not.toThrow(); + }); + + it('close() should do nothing if file descriptor is not open', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + expect(() => sink.close()).not.toThrow(); + }); + + it('write() should write encoded input to file when sink is open', () => { + const sink = new FileSink({ filePath: '/tmp/write-open-unique-test.txt' }); + sink.open(); + const str = 'test data'; + sink.write(str); + expect(fs.readFileSync('/tmp/write-open-unique-test.txt', 'utf8')).toBe( + `${str}\n`, + ); + }); + + it('write() should silently ignore writes when file descriptor is not open', () => { + const sink = new FileSink({ filePath: '/tmp/write-test-closed.txt' }); + expect(() => sink.write('test data')).not.toThrow(); + }); + + it('write() should silently ignore write errors when fs.writeSync throws', () => { + const sink = new FileSink({ filePath: '/tmp/write-error-test.txt' }); + sink.open(); + + // Mock fs.writeSync to throw an error + const writeSyncSpy = vi.spyOn(fs, 'writeSync').mockImplementation(() => { + throw new Error('Write error'); + }); + + try { + // This should not throw despite the write error + expect(() => sink.write('test data')).not.toThrow(); + } finally { + // Restore original function + writeSyncSpy.mockRestore(); + sink.close(); + } + }); + + it('recover() should call the recover function from options', () => { + const mockRecover = vi + .fn() + .mockReturnValue({ records: ['test'], errors: [], partialTail: null }); + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + recover: mockRecover, + }); + expect(sink.recover()).toStrictEqual({ + records: ['test'], + errors: [], + partialTail: null, + }); + expect(mockRecover).toHaveBeenCalledWith(); + }); + + it('repack() should recover records and write them to output path', () => { + const mockRecover = vi.fn(); + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + recover: mockRecover, + }); + const records = ['record1', 'record2']; + mockRecover.mockReturnValue({ records, errors: [], partialTail: null }); + const outputPath = '/tmp/repack-output.txt'; + sink.repack(outputPath); + expect(mockRecover).toHaveBeenCalled(); + expect(fs.readFileSync(outputPath, 'utf8')).toBe('record1\n\nrecord2\n'); + }); + + it('finalize() should call the finalize function from options', () => { + const mockFinalize = vi.fn(); + const sink = new FileSink({ + filePath: '/tmp/test-file.txt', + finalize: mockFinalize, + }); + sink.finalize(); + expect(mockFinalize).toHaveBeenCalledTimes(1); + }); + + it('isClosed() should return true when sink is not opened', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + expect(sink.isClosed()).toBe(true); + }); + + it('isClosed() should return false when sink is opened', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(sink.isClosed()).toBe(false); + }); + + it('isClosed() should return true when sink is closed after being opened', () => { + const sink = new FileSink({ filePath: '/tmp/test-file.txt' }); + sink.open(); + expect(sink.isClosed()).toBe(false); + sink.close(); + expect(sink.isClosed()).toBe(true); + }); +}); diff --git a/packages/utils/src/lib/sink-source.types.ts b/packages/utils/src/lib/sink-source.types.ts new file mode 100644 index 000000000..4473026d0 --- /dev/null +++ b/packages/utils/src/lib/sink-source.types.ts @@ -0,0 +1,48 @@ +export type Encoder = { + encode: (input: I) => O; +}; + +export type Decoder = { + decode: (output: O) => I; +}; + +export type Sink = { + open: () => void; + write: (input: I) => void; + close: () => void; + isClosed: () => boolean; +} & Encoder; + +export type Buffered = { + flush: () => void; +}; +export type BufferedSink = {} & Sink & Buffered; + +export type Source = { + read?: () => O; + decode?: (input: I) => O; +}; + +export type Observer = { + subscribe: () => void; + unsubscribe: () => void; + isSubscribed: () => boolean; +}; + +export type Recoverable = { + recover: () => RecoverResult; + repack: (outputPath?: string) => void; + finalize: () => void; +}; + +export type RecoverResult = { + records: T[]; + errors: { lineNo: number; line: string; error: Error }[]; + partialTail: string | null; +}; + +export type RecoverOptions = { + keepInvalid?: boolean; +}; + +export type Output = {} & BufferedSink; From 756f8c0db48a28ec83126fea624ceb3acf60e6f6 Mon Sep 17 00:00:00 2001 From: John Doe Date: Fri, 9 Jan 2026 22:38:09 +0100 Subject: [PATCH 2/7] feat: add file sink classes --- ...nt.test.ts => file-sink-jsonl.int.test.ts} | 2 +- .../{file-sink-json.ts => file-sink-jsonl.ts} | 0 ...t.test.ts => file-sink-jsonl.unit.test.ts} | 28 ++++++++++++++++++- .../utils/src/lib/file-sink-text.unit.test.ts | 15 ++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) rename packages/utils/src/lib/{file-sink-json.int.test.ts => file-sink-jsonl.int.test.ts} (99%) rename packages/utils/src/lib/{file-sink-json.ts => file-sink-jsonl.ts} (100%) rename packages/utils/src/lib/{file-sink-json.unit.test.ts => file-sink-jsonl.unit.test.ts} (89%) diff --git a/packages/utils/src/lib/file-sink-json.int.test.ts b/packages/utils/src/lib/file-sink-jsonl.int.test.ts similarity index 99% rename from packages/utils/src/lib/file-sink-json.int.test.ts rename to packages/utils/src/lib/file-sink-jsonl.int.test.ts index c331d8d0a..e0f57bbaa 100644 --- a/packages/utils/src/lib/file-sink-json.int.test.ts +++ b/packages/utils/src/lib/file-sink-jsonl.int.test.ts @@ -3,7 +3,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { teardownTestFolder } from '@code-pushup/test-utils'; -import { JsonlFileSink, recoverJsonlFile } from './file-sink-json.js'; +import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; describe('JsonlFileSink integration', () => { const baseDir = path.join(os.tmpdir(), 'file-sink-json-int-tests'); diff --git a/packages/utils/src/lib/file-sink-json.ts b/packages/utils/src/lib/file-sink-jsonl.ts similarity index 100% rename from packages/utils/src/lib/file-sink-json.ts rename to packages/utils/src/lib/file-sink-jsonl.ts diff --git a/packages/utils/src/lib/file-sink-json.unit.test.ts b/packages/utils/src/lib/file-sink-jsonl.unit.test.ts similarity index 89% rename from packages/utils/src/lib/file-sink-json.unit.test.ts rename to packages/utils/src/lib/file-sink-jsonl.unit.test.ts index a920c8cbe..75f981cb0 100644 --- a/packages/utils/src/lib/file-sink-json.unit.test.ts +++ b/packages/utils/src/lib/file-sink-jsonl.unit.test.ts @@ -7,7 +7,7 @@ import { jsonlDecode, jsonlEncode, recoverJsonlFile, -} from './file-sink-json.js'; +} from './file-sink-jsonl.js'; describe('jsonlEncode', () => { it('should encode object to JSON string', () => { @@ -207,10 +207,36 @@ describe('JsonlFileSink', () => { `${records.map(record => JSON.stringify(record)).join('\n')}\n`, ); + sink.repack(); + expect(fs.readFileSync(filePath, 'utf8')).toBe( + `${JSON.stringify(records[0])}\n${JSON.stringify(records[1])}\n`, + ); + }); + + it('repack() should accept output path', () => { + const filePath = '/tmp/jsonl-repack-test.jsonl'; + const sink = new JsonlFileSink({ filePath }); + const records = [ + { key: 'value', number: 42 }, + { key: 'value', number: 42 }, + ]; + + fs.writeFileSync( + filePath, + `${records.map(record => JSON.stringify(record)).join('\n')}\n`, + ); + const outputPath = '/tmp/jsonl-repack-output.jsonl'; sink.repack(outputPath); expect(fs.readFileSync(outputPath, 'utf8')).toBe( `${JSON.stringify(records[0])}\n${JSON.stringify(records[1])}\n`, ); }); + + it('should do nothing on finalize()', () => { + const sink = new JsonlFileSink({ + filePath: '/tmp/jsonl-finalize-test.jsonl', + }); + expect(() => sink.finalize()).not.toThrow(); + }); }); diff --git a/packages/utils/src/lib/file-sink-text.unit.test.ts b/packages/utils/src/lib/file-sink-text.unit.test.ts index f76cf13d4..33cc9ad0e 100644 --- a/packages/utils/src/lib/file-sink-text.unit.test.ts +++ b/packages/utils/src/lib/file-sink-text.unit.test.ts @@ -251,6 +251,21 @@ describe('FileSink', () => { }); it('repack() should recover records and write them to output path', () => { + const mockRecover = vi.fn(); + const filePath = '/tmp/test-file.txt'; + const sink = new FileSink({ + filePath, + recover: mockRecover, + }); + const records = ['record1', 'record2']; + mockRecover.mockReturnValue({ records, errors: [], partialTail: null }); + + sink.repack(); + expect(mockRecover).toHaveBeenCalled(); + expect(fs.readFileSync(filePath, 'utf8')).toBe('record1\n\nrecord2\n'); + }); + + it('repack() should accept output path', () => { const mockRecover = vi.fn(); const sink = new FileSink({ filePath: '/tmp/test-file.txt', From b0c9cc4d9238a10ce979dbde284b9f30329bf4c2 Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 13 Jan 2026 00:01:27 +0100 Subject: [PATCH 3/7] refactor: add trace json file --- .../utils/src/lib/file-sink-json-trace.ts | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 packages/utils/src/lib/file-sink-json-trace.ts diff --git a/packages/utils/src/lib/file-sink-json-trace.ts b/packages/utils/src/lib/file-sink-json-trace.ts new file mode 100644 index 000000000..7933d318c --- /dev/null +++ b/packages/utils/src/lib/file-sink-json-trace.ts @@ -0,0 +1,167 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; +import { getCompleteEvent, getStartTracing } from './trace-file-utils.js'; +import type { + InstantEvent, + SpanEvent, + TraceEvent, + TraceEventRaw, + UserTimingDetail, +} from './trace-file.type.js'; + +const tryJson = (v: unknown): T | unknown => { + if (typeof v !== 'string') return v; + try { + return JSON.parse(v) as T; + } catch { + return v; + } +}; + +const toJson = (v: unknown): unknown => { + if (v === undefined) return undefined; + try { + return JSON.stringify(v); + } catch { + return v; + } +}; + +export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { + if (!args) return rest as TraceEvent; + + const out: any = { ...args }; + if ('detail' in out) out.detail = tryJson(out.detail); + if (out.data?.detail) + out.data.detail = tryJson(out.data.detail); + + return { ...rest, args: out } as TraceEvent; +} + +export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { + if (!args) return rest as TraceEventRaw; + + const out: any = { ...args }; + if ('detail' in out) out.detail = toJson(out.detail); + if (out.data?.detail) out.data.detail = toJson(out.data.detail); + + return { ...rest, args: out } as TraceEventRaw; +} + +function getTraceMetadata( + startDate?: Date, + metadata?: Record, +) { + return { + source: 'DevTools', + startTime: startDate?.toISOString() ?? new Date().toISOString(), + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + ...metadata, + }; +} + +function createTraceFileContent( + traceEventsContent: string, + startDate?: Date, + metadata?: Record, +): string { + return `{ + "metadata": ${JSON.stringify(getTraceMetadata(startDate, metadata))}, + "traceEvents": [ +${traceEventsContent} + ] +}`; +} + +function finalizeTraceFile( + events: (SpanEvent | InstantEvent)[], + outputPath: string, + metadata?: Record, +): void { + const { writeFileSync } = fs; + + const sortedEvents = events.sort((a, b) => a.ts - b.ts); + const first = sortedEvents[0]; + const last = sortedEvents[sortedEvents.length - 1]; + + // Use performance.now() as fallback when no events exist + const fallbackTs = performance.now(); + const firstTs = first?.ts ?? fallbackTs; + const lastTs = last?.ts ?? fallbackTs; + + // Add margins for readability + const tsMargin = 1000; + const startTs = firstTs - tsMargin; + const endTs = lastTs + tsMargin; + const startDate = new Date().toISOString(); + + const traceEventsJson = [ + // Preamble + encodeTraceEvent( + getStartTracing({ + ts: startTs, + url: outputPath, + }), + ), + encodeTraceEvent( + getCompleteEvent({ + ts: startTs, + dur: 20, + }), + ), + // Events + ...events.map(encodeTraceEvent), + encodeTraceEvent( + getCompleteEvent({ + ts: endTs, + dur: 20, + }), + ), + ].join(',\n'); + + const jsonOutput = createTraceFileContent( + traceEventsJson, + new Date(), + metadata, + ); + writeFileSync(outputPath, jsonOutput, 'utf8'); +} + +export interface TraceFileSinkOptions { + filename: string; + directory?: string; + metadata?: Record; +} + +export class TraceFileSink extends JsonlFileSink { + readonly #filePath: string; + readonly #getFilePathForExt: (ext: 'json' | 'jsonl') => string; + readonly #metadata: Record | undefined; + + constructor(opts: TraceFileSinkOptions) { + const { filename, directory = '.', metadata } = opts; + + const traceJsonlPath = path.join(directory, `${filename}.jsonl`); + + super({ + filePath: traceJsonlPath, + recover: () => recoverJsonlFile(traceJsonlPath), + }); + + this.#metadata = metadata; + this.#filePath = path.join(directory, `${filename}.json`); + this.#getFilePathForExt = (ext: 'json' | 'jsonl') => + path.join(directory, `${filename}.${ext}`); + } + + override finalize(): void { + finalizeTraceFile(this.recover().records, this.#filePath, this.#metadata); + } + + getFilePathForExt(ext: 'json' | 'jsonl'): string { + return this.#getFilePathForExt(ext); + } +} From 1f6e32628e5a2aeffea6c5c560acb60868abe324 Mon Sep 17 00:00:00 2001 From: John Doe Date: Wed, 14 Jan 2026 01:05:15 +0100 Subject: [PATCH 4/7] refactor: wip --- packages/utils/src/lib/file-sink-text.ts | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/utils/src/lib/file-sink-text.ts b/packages/utils/src/lib/file-sink-text.ts index 3cafacbe4..050188e58 100644 --- a/packages/utils/src/lib/file-sink-text.ts +++ b/packages/utils/src/lib/file-sink-text.ts @@ -1,6 +1,7 @@ import * as fs from 'node:fs'; import { existsSync, mkdirSync } from 'node:fs'; import path from 'node:path'; +import { PROFILER_FILE_BASE_NAME, PROFILER_OUT_DIR } from './profiler'; import type { RecoverOptions, RecoverResult, @@ -60,6 +61,42 @@ export const stringRecover = function ( return { records, errors, partialTail }; }; +export type FileNameOptions = { + fileBaseName: string; + outDir: string; + fileName?: string; +}; + +export function getFilenameParts(options: FileNameOptions): { + outDir: string; + fileName: string; +} { + const { fileName, fileBaseName, outDir } = options; + + if (fileName) { + return { + outDir, + fileName, + }; + } + + const baseName = fileBaseName; + const DATE_LENGTH = 10; + const TIME_SEGMENTS = 3; + const COLON_LENGTH = 1; + const TOTAL_TIME_LENGTH = + TIME_SEGMENTS * 2 + (TIME_SEGMENTS - 1) * COLON_LENGTH; // HH:MM:SS = 8 chars + const id = new Date() + .toISOString() + .slice(0, DATE_LENGTH + TOTAL_TIME_LENGTH) + .replace(/:/g, '-'); + + return { + outDir, + fileName: `${baseName}.${id}`, + }; +} + export type FileSinkOptions = { filePath: string; recover?: () => RecoverResult; From 6bcb73b7e6d9c967cef339d371e56a99df0fc4c8 Mon Sep 17 00:00:00 2001 From: John Doe Date: Wed, 14 Jan 2026 02:23:33 +0100 Subject: [PATCH 5/7] refactor: wip --- .../src/lib/file-sink-json-trace.int.test.ts | 224 ++++++++++++ .../utils/src/lib/file-sink-json-trace.ts | 88 +++-- .../src/lib/file-sink-json-trace.unit.test.ts | 335 ++++++++++++++++++ 3 files changed, 613 insertions(+), 34 deletions(-) create mode 100644 packages/utils/src/lib/file-sink-json-trace.int.test.ts create mode 100644 packages/utils/src/lib/file-sink-json-trace.unit.test.ts diff --git a/packages/utils/src/lib/file-sink-json-trace.int.test.ts b/packages/utils/src/lib/file-sink-json-trace.int.test.ts new file mode 100644 index 000000000..e71bb90d5 --- /dev/null +++ b/packages/utils/src/lib/file-sink-json-trace.int.test.ts @@ -0,0 +1,224 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { teardownTestFolder } from '@code-pushup/test-utils'; +import { TraceFileSink } from './file-sink-json-trace.js'; +import type { TraceEvent } from './trace-file.type'; + +describe('TraceFileSink integration', () => { + const baseDir = path.join(os.tmpdir(), 'file-sink-json-trace-int-tests'); + const traceJsonPath = path.join(baseDir, 'test-data.json'); + const traceJsonlPath = path.join(baseDir, 'test-data.jsonl'); + + beforeAll(async () => { + await fs.promises.mkdir(baseDir, { recursive: true }); + }); + + beforeEach(async () => { + try { + await fs.promises.unlink(traceJsonPath); + } catch { + // File doesn't exist, which is fine + } + try { + await fs.promises.unlink(traceJsonlPath); + } catch { + // File doesn't exist, which is fine + } + }); + + afterAll(async () => { + await teardownTestFolder(baseDir); + }); + + describe('file operations', () => { + const testEvents: TraceEvent[] = [ + { name: 'navigationStart', ts: 100, ph: 'I', cat: 'blink.user_timing' }, + { + name: 'loadEventStart', + ts: 200, + ph: 'I', + cat: 'blink.user_timing', + args: { data: { url: 'https://example.com' } }, + }, + { + name: 'loadEventEnd', + ts: 250, + ph: 'I', + cat: 'blink.user_timing', + args: { detail: { duration: 50 } }, + }, + ]; + + it('should write and read trace events', async () => { + const sink = new TraceFileSink({ + filename: 'test-data', + directory: baseDir, + }); + + // Open and write data + sink.open(); + testEvents.forEach(event => sink.write(event as any)); + sink.finalize(); + + expect(fs.existsSync(traceJsonPath)).toBe(true); + expect(fs.existsSync(traceJsonlPath)).toBe(true); + + const jsonContent = fs.readFileSync(traceJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect(traceData.metadata.source).toBe('DevTools'); + expect(traceData.metadata.dataOrigin).toBe('TraceEvents'); + expect(Array.isArray(traceData.traceEvents)).toBe(true); + + // Should have preamble events + user events + complete event + expect(traceData.traceEvents.length).toBeGreaterThan(testEvents.length); + + // Check that our events are included + const userEvents = traceData.traceEvents.filter((e: any) => + testEvents.some(testEvent => testEvent.name === e.name), + ); + expect(userEvents).toHaveLength(testEvents.length); + }); + + it('should recover events from JSONL file', async () => { + const sink = new TraceFileSink({ + filename: 'test-data', + directory: baseDir, + }); + sink.open(); + testEvents.forEach(event => sink.write(event as any)); + sink.close(); + + const recovered = sink.recover(); + expect(recovered.records).toStrictEqual(testEvents); + expect(recovered.errors).toStrictEqual([]); + expect(recovered.partialTail).toBeNull(); + }); + + it('should handle empty trace files', async () => { + const sink = new TraceFileSink({ + filename: 'empty-test', + directory: baseDir, + }); + sink.open(); + sink.finalize(); + + const emptyJsonPath = path.join(baseDir, 'empty-test.json'); + expect(fs.existsSync(emptyJsonPath)).toBe(true); + + const jsonContent = fs.readFileSync(emptyJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect(traceData.metadata.source).toBe('DevTools'); + // Should have at least preamble and complete events + expect(traceData.traceEvents.length).toBeGreaterThanOrEqual(2); + }); + + it('should handle metadata in trace files', async () => { + const metadata = { + version: '1.0.0', + environment: 'test', + customData: { key: 'value' }, + }; + + const sink = new TraceFileSink({ + filename: 'metadata-test', + directory: baseDir, + metadata, + }); + sink.open(); + sink.write({ name: 'test-event', ts: 100, ph: 'I' } as any); + sink.finalize(); + + const metadataJsonPath = path.join(baseDir, 'metadata-test.json'); + const jsonContent = fs.readFileSync(metadataJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect(traceData.metadata.version).toBe('1.0.0'); + expect(traceData.metadata.environment).toBe('test'); + expect(traceData.metadata.customData).toStrictEqual({ key: 'value' }); + expect(traceData.metadata.source).toBe('DevTools'); + }); + + describe('edge cases', () => { + it('should handle single event traces', async () => { + const singleEvent: TraceEvent = { + name: 'singleEvent', + ts: 123, + ph: 'I', + cat: 'test', + }; + + const sink = new TraceFileSink({ + filename: 'single-event-test', + directory: baseDir, + }); + sink.open(); + sink.write(singleEvent as any); + sink.finalize(); + + const singleJsonPath = path.join(baseDir, 'single-event-test.json'); + const jsonContent = fs.readFileSync(singleJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + expect( + traceData.traceEvents.some((e: any) => e.name === 'singleEvent'), + ).toBe(true); + }); + + it('should handle events with complex args', async () => { + const complexEvent: TraceEvent = { + name: 'complexEvent', + ts: 456, + ph: 'X', + cat: 'test', + args: { + detail: { nested: { data: [1, 2, 3] } }, + data: { url: 'https://example.com', size: 1024 }, + }, + }; + + const sink = new TraceFileSink({ + filename: 'complex-args-test', + directory: baseDir, + }); + sink.open(); + sink.write(complexEvent as any); + sink.finalize(); + + const complexJsonPath = path.join(baseDir, 'complex-args-test.json'); + const jsonContent = fs.readFileSync(complexJsonPath, 'utf8'); + const traceData = JSON.parse(jsonContent); + + const eventInTrace = traceData.traceEvents.find( + (e: any) => e.name === 'complexEvent', + ); + expect(eventInTrace).toBeDefined(); + expect(eventInTrace.args.detail).toStrictEqual( + '{"nested":{"data":[1,2,3]}}', + ); + expect(eventInTrace.args.data.url).toBe('https://example.com'); + }); + + it('should handle non-existent directories gracefully', async () => { + const nonExistentDir = path.join(baseDir, 'non-existent'); + const sink = new TraceFileSink({ + filename: 'non-existent-dir-test', + directory: nonExistentDir, + }); + + sink.open(); + sink.write({ name: 'test', ts: 100, ph: 'I' } as any); + sink.finalize(); + + const jsonPath = path.join( + nonExistentDir, + 'non-existent-dir-test.json', + ); + expect(fs.existsSync(jsonPath)).toBe(true); + }); + }); + }); +}); diff --git a/packages/utils/src/lib/file-sink-json-trace.ts b/packages/utils/src/lib/file-sink-json-trace.ts index 7933d318c..f35895303 100644 --- a/packages/utils/src/lib/file-sink-json-trace.ts +++ b/packages/utils/src/lib/file-sink-json-trace.ts @@ -1,7 +1,12 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { performance } from 'node:perf_hooks'; -import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; +import { + JsonlFileSink, + jsonlDecode, + jsonlEncode, + recoverJsonlFile, +} from './file-sink-jsonl.js'; import { getCompleteEvent, getStartTracing } from './trace-file-utils.js'; import type { InstantEvent, @@ -11,46 +16,60 @@ import type { UserTimingDetail, } from './trace-file.type.js'; -const tryJson = (v: unknown): T | unknown => { - if (typeof v !== 'string') return v; - try { - return JSON.parse(v) as T; - } catch { - return v; +export function decodeDetail(target: UserTimingDetail): UserTimingDetail { + if (typeof target.detail === 'string') { + return { ...target, detail: jsonlDecode(target.detail) }; } -}; + return target; +} -const toJson = (v: unknown): unknown => { - if (v === undefined) return undefined; - try { - return JSON.stringify(v); - } catch { - return v; +export function encodeDetail(target: UserTimingDetail): UserTimingDetail { + if (target.detail && typeof target.detail === 'object') { + return { + ...target, + detail: jsonlEncode(target.detail as UserTimingDetail), + }; } -}; + return target; +} export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { if (!args) return rest as TraceEvent; - const out: any = { ...args }; - if ('detail' in out) out.detail = tryJson(out.detail); - if (out.data?.detail) - out.data.detail = tryJson(out.data.detail); + const out: UserTimingDetail = { ...args }; + const processedOut = decodeDetail(out); - return { ...rest, args: out } as TraceEvent; + return { + ...rest, + args: + out.data && typeof out.data === 'object' + ? { + ...processedOut, + data: decodeDetail(out.data as UserTimingDetail), + } + : processedOut, + }; } export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { if (!args) return rest as TraceEventRaw; - const out: any = { ...args }; - if ('detail' in out) out.detail = toJson(out.detail); - if (out.data?.detail) out.data.detail = toJson(out.data.detail); + const out: UserTimingDetail = { ...args }; + const processedOut = encodeDetail(out); - return { ...rest, args: out } as TraceEventRaw; + return { + ...rest, + args: + out.data && typeof out.data === 'object' + ? { + ...processedOut, + data: encodeDetail(out.data as UserTimingDetail), + } + : processedOut, + }; } -function getTraceMetadata( +export function getTraceMetadata( startDate?: Date, metadata?: Record, ) { @@ -76,30 +95,30 @@ ${traceEventsContent} }`; } -function finalizeTraceFile( +export function finalizeTraceFile( events: (SpanEvent | InstantEvent)[], outputPath: string, metadata?: Record, ): void { const { writeFileSync } = fs; + if (events.length === 0) { + return; + } + const sortedEvents = events.sort((a, b) => a.ts - b.ts); const first = sortedEvents[0]; const last = sortedEvents[sortedEvents.length - 1]; - // Use performance.now() as fallback when no events exist const fallbackTs = performance.now(); const firstTs = first?.ts ?? fallbackTs; const lastTs = last?.ts ?? fallbackTs; - // Add margins for readability const tsMargin = 1000; const startTs = firstTs - tsMargin; const endTs = lastTs + tsMargin; - const startDate = new Date().toISOString(); const traceEventsJson = [ - // Preamble encodeTraceEvent( getStartTracing({ ts: startTs, @@ -112,7 +131,6 @@ function finalizeTraceFile( dur: 20, }), ), - // Events ...events.map(encodeTraceEvent), encodeTraceEvent( getCompleteEvent({ @@ -120,7 +138,9 @@ function finalizeTraceFile( dur: 20, }), ), - ].join(',\n'); + ] + .map(event => JSON.stringify(event)) + .join(',\n'); const jsonOutput = createTraceFileContent( traceEventsJson, @@ -130,11 +150,11 @@ function finalizeTraceFile( writeFileSync(outputPath, jsonOutput, 'utf8'); } -export interface TraceFileSinkOptions { +export type TraceFileSinkOptions = { filename: string; directory?: string; metadata?: Record; -} +}; export class TraceFileSink extends JsonlFileSink { readonly #filePath: string; diff --git a/packages/utils/src/lib/file-sink-json-trace.unit.test.ts b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts new file mode 100644 index 000000000..162f5f048 --- /dev/null +++ b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts @@ -0,0 +1,335 @@ +import { vol } from 'memfs'; +import * as fs from 'node:fs'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { + TraceFileSink, + decodeTraceEvent, + encodeTraceEvent, + finalizeTraceFile, + getTraceMetadata, +} from './file-sink-json-trace.js'; +import type { + InstantEvent, + TraceEvent, + TraceEventRaw, +} from './trace-file.type'; + +describe('decodeTraceEvent', () => { + it('should return event without args if no args present', () => { + const event: TraceEventRaw = { name: 'test', ts: 123 }; + expect(decodeTraceEvent(event)).toStrictEqual(event); + }); + + it('should decode args with detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: '{"key":"value"}' }, + }; + expect(decodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { detail: { key: 'value' } }, + }); + }); + + it('should decode nested data.detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { data: { detail: '{"nested":"value"}' } }, + }; + expect(decodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { data: { detail: { nested: 'value' } } }, + }); + }); + + it('should handle invalid JSON in detail', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: 'invalid json' }, + }; + expect(() => decodeTraceEvent(event)).toThrow('Unexpected token'); + }); +}); + +describe('encodeTraceEvent', () => { + it('should return event without args if no args present', () => { + const event: TraceEventRaw = { name: 'test', ts: 123 }; + expect(encodeTraceEvent(event)).toStrictEqual(event); + }); + + it('should encode args with detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: { key: 'value' } }, + }; + expect(encodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { detail: '{"key":"value"}' }, + }); + }); + + it('should encode nested data.detail property', () => { + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { data: { detail: { nested: 'value' } } }, + }; + expect(encodeTraceEvent(event)).toStrictEqual({ + name: 'test', + ts: 123, + args: { data: { detail: '{"nested":"value"}' } }, + }); + }); + + it('should handle non-serializable detail', () => { + const circular: any = {}; + circular.self = circular; + const event: TraceEventRaw = { + name: 'test', + ts: 123, + args: { detail: circular }, + }; + expect(() => encodeTraceEvent(event)).toThrow( + 'Converting circular structure to JSON', + ); + }); +}); + +describe('finalizeTraceFile', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + it('should create trace file with events', () => { + const events: TraceEvent[] = [ + { name: 'event1', ts: 100, ph: 'I' }, + { name: 'event2', ts: 200, ph: 'X', args: { dur: 50 } }, + ]; + const outputPath = '/tmp/test-trace.json'; + + finalizeTraceFile(events as any, outputPath); + + expect(fs.existsSync(outputPath)).toBe(true); + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + expect(content.metadata.source).toBe('DevTools'); + expect(content.traceEvents).toHaveLength(5); // preamble (start + complete) + events + complete + }); + + it('should handle empty events array', () => { + const events: TraceEvent[] = []; + const outputPath = '/tmp/empty-trace.json'; + + finalizeTraceFile(events as any, outputPath); + + expect(fs.existsSync(outputPath)).toBe(true); + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + expect(content.traceEvents).toHaveLength(3); // preamble (start + complete) + end complete + }); + + it('should sort events by timestamp', () => { + const events: TraceEvent[] = [ + { name: 'event2', ts: 200, ph: 'I' }, + { name: 'event1', ts: 100, ph: 'I' }, + ]; + const outputPath = '/tmp/sorted-trace.json'; + + finalizeTraceFile(events as any, outputPath); + + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + const eventNames = content.traceEvents + .filter((e: any) => e.name.startsWith('event')) + .map((e: any) => e.name); + expect(eventNames).toStrictEqual(['event1', 'event2']); + }); +}); + +describe('TraceFileSink', () => { + beforeEach(() => { + vol.fromJSON( + { + '/tmp': null, + }, + MEMFS_VOLUME, + ); + }); + + it('should create trace file sink with default options', () => { + const sink = new TraceFileSink({ filename: 'test' }); + expect(sink.getFilePathForExt('json')).toBe('test.json'); + expect(sink.getFilePathForExt('jsonl')).toBe('test.jsonl'); + }); + + it('should create trace file sink with custom directory', () => { + const sink = new TraceFileSink({ + filename: 'test', + directory: '/tmp/custom', + }); + expect(sink.getFilePathForExt('json')).toBe('/tmp/custom/test.json'); + expect(sink.getFilePathForExt('jsonl')).toBe('/tmp/custom/test.jsonl'); + }); + + it('should handle file operations with trace events', () => { + const sink = new TraceFileSink({ + filename: 'trace-test', + directory: '/tmp', + }); + sink.open(); + + const event1: InstantEvent = { name: 'mark1', ts: 100, ph: 'I' }; + const event2: InstantEvent = { name: 'mark2', ts: 200, ph: 'I' }; + sink.write(event1); + sink.write(event2); + sink.close(); + + expect(fs.existsSync('/tmp/trace-test.jsonl')).toBe(true); + expect(fs.existsSync('/tmp/trace-test.json')).toBe(false); + + const recovered = sink.recover(); + expect(recovered.records).toStrictEqual([event1, event2]); + }); + + it('should create trace file on finalize', () => { + const sink = new TraceFileSink({ + filename: 'finalize-test', + directory: '/tmp', + }); + sink.open(); + + const event: InstantEvent = { name: 'test-event', ts: 150, ph: 'I' }; + sink.write(event); + sink.finalize(); + + expect(fs.existsSync('/tmp/finalize-test.json')).toBe(true); + const content = JSON.parse( + fs.readFileSync('/tmp/finalize-test.json', 'utf8'), + ); + expect(content.metadata.source).toBe('DevTools'); + expect(content.traceEvents.some((e: any) => e.name === 'test-event')).toBe( + true, + ); + }); + + it('should handle metadata in finalize', () => { + const metadata = { customField: 'value', version: '1.0' }; + const sink = new TraceFileSink({ + filename: 'metadata-test', + directory: '/tmp', + metadata, + }); + sink.open(); + sink.write({ name: 'event', ts: 100, ph: 'I' }); + sink.finalize(); + + const content = JSON.parse( + fs.readFileSync('/tmp/metadata-test.json', 'utf8'), + ); + expect(content.metadata.customField).toBe('value'); + expect(content.metadata.version).toBe('1.0'); + }); + + it('should do nothing on finalize when no events written', () => { + const sink = new TraceFileSink({ + filename: 'empty-test', + directory: '/tmp', + }); + sink.open(); + sink.finalize(); + + expect(fs.existsSync('/tmp/empty-test.json')).toBe(true); + const content = JSON.parse(fs.readFileSync('/tmp/empty-test.json', 'utf8')); + expect(content.traceEvents).toHaveLength(3); // preamble (start + complete) + end complete + }); +}); + +describe('getTraceMetadata', () => { + it('should use provided startDate when given', () => { + const startDate = new Date('2023-01-15T10:30:00.000Z'); + const metadata = { customField: 'value' }; + + const result = getTraceMetadata(startDate, metadata); + + expect(result).toStrictEqual({ + source: 'DevTools', + startTime: '2023-01-15T10:30:00.000Z', + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + customField: 'value', + }); + }); + + it('should use current date when startDate is undefined', () => { + const beforeTest = new Date(); + const metadata = { version: '1.0' }; + + const result = getTraceMetadata(undefined, metadata); + + const afterTest = new Date(); + expect(result.source).toBe('DevTools'); + expect(result.hardwareConcurrency).toBe(1); + expect(result.dataOrigin).toBe('TraceEvents'); + + // Verify startTime is a valid ISO string between test execution + const startTime = new Date(result.startTime); + expect(startTime.getTime()).toBeGreaterThanOrEqual(beforeTest.getTime()); + expect(startTime.getTime()).toBeLessThanOrEqual(afterTest.getTime()); + }); + + it('should use current date when startDate is null', () => { + const beforeTest = new Date(); + const metadata = { environment: 'test' }; + + const result = getTraceMetadata(undefined, metadata); + + const afterTest = new Date(); + expect(result.source).toBe('DevTools'); + expect(result.hardwareConcurrency).toBe(1); + expect(result.dataOrigin).toBe('TraceEvents'); + + // Verify startTime is a valid ISO string between test execution + const startTime = new Date(result.startTime); + expect(startTime.getTime()).toBeGreaterThanOrEqual(beforeTest.getTime()); + expect(startTime.getTime()).toBeLessThanOrEqual(afterTest.getTime()); + }); + + it('should handle empty metadata', () => { + const startDate = new Date('2023-12-25T00:00:00.000Z'); + + const result = getTraceMetadata(startDate); + + expect(result).toStrictEqual({ + source: 'DevTools', + startTime: '2023-12-25T00:00:00.000Z', + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + }); + }); + + it('should handle both startDate and metadata undefined', () => { + const beforeTest = new Date(); + + const result = getTraceMetadata(); + + const afterTest = new Date(); + expect(result.source).toBe('DevTools'); + expect(result.hardwareConcurrency).toBe(1); + expect(result.dataOrigin).toBe('TraceEvents'); + + // Verify startTime is a valid ISO string between test execution + const startTime = new Date(result.startTime); + expect(startTime.getTime()).toBeGreaterThanOrEqual(beforeTest.getTime()); + expect(startTime.getTime()).toBeLessThanOrEqual(afterTest.getTime()); + }); +}); From 3fe68717586616c645153e177d91d648e2bf231f Mon Sep 17 00:00:00 2001 From: Michael Hladky Date: Wed, 14 Jan 2026 22:53:32 +0100 Subject: [PATCH 6/7] refactor: wip arc --- .../src/lib/file-sink-json-trace.int.test.ts | 26 +- .../utils/src/lib/file-sink-json-trace.ts | 294 +++++++++--------- .../src/lib/file-sink-json-trace.unit.test.ts | 99 ++++-- .../utils/src/lib/file-sink-jsonl.int.test.ts | 8 +- packages/utils/src/lib/file-sink-jsonl.ts | 177 +++++++++-- .../src/lib/file-sink-jsonl.unit.test.ts | 17 +- packages/utils/src/lib/file-sink-text.ts | 291 ++++++++++------- .../utils/src/lib/file-sink-text.unit.test.ts | 4 +- packages/utils/src/lib/trace-file-utils.ts | 80 ++++- .../src/lib/utils/perf-hooks.mock.ts | 33 +- 10 files changed, 681 insertions(+), 348 deletions(-) diff --git a/packages/utils/src/lib/file-sink-json-trace.int.test.ts b/packages/utils/src/lib/file-sink-json-trace.int.test.ts index e71bb90d5..b9cdb7f2e 100644 --- a/packages/utils/src/lib/file-sink-json-trace.int.test.ts +++ b/packages/utils/src/lib/file-sink-json-trace.int.test.ts @@ -3,8 +3,8 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { teardownTestFolder } from '@code-pushup/test-utils'; -import { TraceFileSink } from './file-sink-json-trace.js'; -import type { TraceEvent } from './trace-file.type'; +import { FileSinkJsonTrace } from './file-sink-json-trace'; +import type { CompleteEvent, TraceEvent } from './trace-file.type'; describe('TraceFileSink integration', () => { const baseDir = path.join(os.tmpdir(), 'file-sink-json-trace-int-tests'); @@ -52,7 +52,7 @@ describe('TraceFileSink integration', () => { ]; it('should write and read trace events', async () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'test-data', directory: baseDir, }); @@ -83,7 +83,7 @@ describe('TraceFileSink integration', () => { }); it('should recover events from JSONL file', async () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'test-data', directory: baseDir, }); @@ -98,7 +98,7 @@ describe('TraceFileSink integration', () => { }); it('should handle empty trace files', async () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'empty-test', directory: baseDir, }); @@ -123,7 +123,7 @@ describe('TraceFileSink integration', () => { customData: { key: 'value' }, }; - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'metadata-test', directory: baseDir, metadata, @@ -151,7 +151,7 @@ describe('TraceFileSink integration', () => { cat: 'test', }; - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'single-event-test', directory: baseDir, }); @@ -169,7 +169,7 @@ describe('TraceFileSink integration', () => { }); it('should handle events with complex args', async () => { - const complexEvent: TraceEvent = { + const complexEvent: CompleteEvent = { name: 'complexEvent', ts: 456, ph: 'X', @@ -180,7 +180,7 @@ describe('TraceFileSink integration', () => { }, }; - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'complex-args-test', directory: baseDir, }); @@ -196,15 +196,15 @@ describe('TraceFileSink integration', () => { (e: any) => e.name === 'complexEvent', ); expect(eventInTrace).toBeDefined(); - expect(eventInTrace.args.detail).toStrictEqual( - '{"nested":{"data":[1,2,3]}}', - ); + expect(eventInTrace.args.detail).toStrictEqual({ + nested: { data: [1, 2, 3] }, + }); expect(eventInTrace.args.data.url).toBe('https://example.com'); }); it('should handle non-existent directories gracefully', async () => { const nonExistentDir = path.join(baseDir, 'non-existent'); - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'non-existent-dir-test', directory: nonExistentDir, }); diff --git a/packages/utils/src/lib/file-sink-json-trace.ts b/packages/utils/src/lib/file-sink-json-trace.ts index f35895303..6e201f133 100644 --- a/packages/utils/src/lib/file-sink-json-trace.ts +++ b/packages/utils/src/lib/file-sink-json-trace.ts @@ -1,187 +1,193 @@ import * as fs from 'node:fs'; +// Exception: finalization creates new JSON file import * as path from 'node:path'; import { performance } from 'node:perf_hooks'; +import { JsonlFile, recoverJsonlFile } from './file-sink-jsonl.js'; +import type { RecoverResult } from './sink-source.type.js'; import { - JsonlFileSink, - jsonlDecode, - jsonlEncode, - recoverJsonlFile, -} from './file-sink-jsonl.js'; -import { getCompleteEvent, getStartTracing } from './trace-file-utils.js'; + decodeTraceEvent, + encodeTraceEvent, + getCompleteEvent, + getInstantEventTracingStartedInBrowser, +} from './trace-file-utils.js'; import type { InstantEvent, SpanEvent, TraceEvent, TraceEventRaw, - UserTimingDetail, } from './trace-file.type.js'; -export function decodeDetail(target: UserTimingDetail): UserTimingDetail { - if (typeof target.detail === 'string') { - return { ...target, detail: jsonlDecode(target.detail) }; - } - return target; -} - -export function encodeDetail(target: UserTimingDetail): UserTimingDetail { - if (target.detail && typeof target.detail === 'object') { - return { - ...target, - detail: jsonlEncode(target.detail as UserTimingDetail), - }; - } - return target; -} - -export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { - if (!args) return rest as TraceEvent; - - const out: UserTimingDetail = { ...args }; - const processedOut = decodeDetail(out); - - return { - ...rest, - args: - out.data && typeof out.data === 'object' - ? { - ...processedOut, - data: decodeDetail(out.data as UserTimingDetail), - } - : processedOut, - }; -} - -export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { - if (!args) return rest as TraceEventRaw; - - const out: UserTimingDetail = { ...args }; - const processedOut = encodeDetail(out); - - return { - ...rest, - args: - out.data && typeof out.data === 'object' - ? { - ...processedOut, - data: encodeDetail(out.data as UserTimingDetail), - } - : processedOut, - }; -} +const TRACE_START_MARGIN_NAME = '[trace padding start]'; +const TRACE_END_MARGIN_NAME = '[trace padding end]'; +const TRACE_MARGIN_MS = 1000; +const TRACE_MARGIN_DURATION_MS = 20; -export function getTraceMetadata( - startDate?: Date, - metadata?: Record, -) { - return { - source: 'DevTools', - startTime: startDate?.toISOString() ?? new Date().toISOString(), - hardwareConcurrency: 1, - dataOrigin: 'TraceEvents', - ...metadata, - }; -} - -function createTraceFileContent( - traceEventsContent: string, - startDate?: Date, - metadata?: Record, -): string { - return `{ - "metadata": ${JSON.stringify(getTraceMetadata(startDate, metadata))}, - "traceEvents": [ -${traceEventsContent} - ] -}`; -} +export type FinalizeTraceFileOptions = { + marginMs?: number; + marginDurMs?: number; + startTime?: string | Date; +}; export function finalizeTraceFile( events: (SpanEvent | InstantEvent)[], outputPath: string, metadata?: Record, + options?: FinalizeTraceFileOptions, ): void { - const { writeFileSync } = fs; - - if (events.length === 0) { - return; - } - - const sortedEvents = events.sort((a, b) => a.ts - b.ts); - const first = sortedEvents[0]; - const last = sortedEvents[sortedEvents.length - 1]; - + events.sort((a, b) => a.ts - b.ts); const fallbackTs = performance.now(); - const firstTs = first?.ts ?? fallbackTs; - const lastTs = last?.ts ?? fallbackTs; - - const tsMargin = 1000; - const startTs = firstTs - tsMargin; - const endTs = lastTs + tsMargin; - - const traceEventsJson = [ - encodeTraceEvent( - getStartTracing({ - ts: startTs, - url: outputPath, - }), - ), - encodeTraceEvent( - getCompleteEvent({ - ts: startTs, - dur: 20, - }), - ), - ...events.map(encodeTraceEvent), - encodeTraceEvent( - getCompleteEvent({ - ts: endTs, - dur: 20, - }), - ), - ] - .map(event => JSON.stringify(event)) - .join(',\n'); - - const jsonOutput = createTraceFileContent( - traceEventsJson, - new Date(), - metadata, + const firstTs = events.length > 0 ? events[0].ts : fallbackTs; + const lastTs = events.length > 0 ? events[events.length - 1].ts : fallbackTs; + + const marginMs = options?.marginMs ?? TRACE_MARGIN_MS; + const marginDurMs = options?.marginDurMs ?? TRACE_MARGIN_DURATION_MS; + + const startTs = firstTs - marginMs; + const endTs = lastTs + marginMs; + + const traceEvents: TraceEvent[] = [ + getInstantEventTracingStartedInBrowser({ ts: startTs, url: outputPath }), + getCompleteEvent({ + name: TRACE_START_MARGIN_NAME, + ts: startTs, + dur: marginDurMs, + }), + ...events, + getCompleteEvent({ + name: TRACE_END_MARGIN_NAME, + ts: endTs, + dur: marginDurMs, + }), + ]; + + const startTime = options?.startTime + ? typeof options.startTime === 'string' + ? options.startTime + : options.startTime.toISOString() + : new Date().toISOString(); + + fs.writeFileSync( + outputPath, + JSON.stringify({ + traceEvents, + displayTimeUnit: 'ms', + metadata: { + source: 'DevTools', + startTime, + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + ...metadata, + }, + }), + 'utf8', ); - writeFileSync(outputPath, jsonOutput, 'utf8'); } export type TraceFileSinkOptions = { filename: string; directory?: string; metadata?: Record; + marginMs?: number; + marginDurMs?: number; + startTime?: string | Date; }; -export class TraceFileSink extends JsonlFileSink { - readonly #filePath: string; - readonly #getFilePathForExt: (ext: 'json' | 'jsonl') => string; +export class FileSinkJsonTrace { + readonly #directory: string; + readonly #filename: string; readonly #metadata: Record | undefined; + readonly #marginMs?: number; + readonly #marginDurMs?: number; + readonly #startTime?: string | Date; + private sink: JsonlFile; + #finalized = false; constructor(opts: TraceFileSinkOptions) { - const { filename, directory = '.', metadata } = opts; - + const { + filename, + directory = '.', + metadata, + marginMs, + marginDurMs, + startTime, + } = opts; const traceJsonlPath = path.join(directory, `${filename}.jsonl`); - super({ + this.#directory = directory; + this.#filename = filename; + this.#metadata = metadata; + this.#marginMs = marginMs; + this.#marginDurMs = marginDurMs; + this.#startTime = startTime; + + this.sink = new JsonlFile({ filePath: traceJsonlPath, - recover: () => recoverJsonlFile(traceJsonlPath), + recover: () => recoverJsonlFile(traceJsonlPath), + finalize: () => { + const rawRecords = this.sink.recover().records; + // Decode raw events to proper TraceEvent format for finalization + const processedRecords = rawRecords.map(decodeTraceEvent); + finalizeTraceFile( + processedRecords as (SpanEvent | InstantEvent)[], + this.getFilePathForExt('json'), + this.#metadata, + { + marginMs: this.#marginMs, + marginDurMs: this.#marginDurMs, + startTime: this.#startTime, + }, + ); + }, }); + } - this.#metadata = metadata; - this.#filePath = path.join(directory, `${filename}.json`); - this.#getFilePathForExt = (ext: 'json' | 'jsonl') => - path.join(directory, `${filename}.${ext}`); + /** + * Open file for writing (no-op since JsonlFile opens lazily). + */ + open(): void { + // JsonlFile opens lazily on first write, so no-op here + } + + write(input: SpanEvent | InstantEvent): void { + const encodedEvent = encodeTraceEvent(input); + this.sink.write(encodedEvent); + } + + /** + * Read all events (strict parsing - throws on invalid JSON). + * For error-tolerant reading, use recover() instead. + */ + readAll(): (SpanEvent | InstantEvent)[] { + return this.sink.readAll().map(decodeTraceEvent) as ( + | SpanEvent + | InstantEvent + )[]; + } + + getFilePath(): string { + return this.sink.getPath(); + } + + close(): void { + this.sink.close(); + } + + recover(): RecoverResult { + const { records, errors, partialTail } = this.sink.recover(); + const processedRecords = records.map(decodeTraceEvent) as ( + | SpanEvent + | InstantEvent + )[]; + return { records: processedRecords, errors, partialTail }; } - override finalize(): void { - finalizeTraceFile(this.recover().records, this.#filePath, this.#metadata); + finalize(): void { + if (this.#finalized) return; + this.#finalized = true; + this.sink.finalize(); } getFilePathForExt(ext: 'json' | 'jsonl'): string { - return this.#getFilePathForExt(ext); + return path.join(this.#directory, `${this.#filename}.${ext}`); } } diff --git a/packages/utils/src/lib/file-sink-json-trace.unit.test.ts b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts index 162f5f048..30911b558 100644 --- a/packages/utils/src/lib/file-sink-json-trace.unit.test.ts +++ b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts @@ -2,13 +2,12 @@ import { vol } from 'memfs'; import * as fs from 'node:fs'; import { beforeEach, describe, expect, it } from 'vitest'; import { MEMFS_VOLUME } from '@code-pushup/test-utils'; +import { FileSinkJsonTrace, finalizeTraceFile } from './file-sink-json-trace'; import { - TraceFileSink, decodeTraceEvent, encodeTraceEvent, - finalizeTraceFile, getTraceMetadata, -} from './file-sink-json-trace.js'; +} from './trace-file-utils.js'; import type { InstantEvent, TraceEvent, @@ -134,9 +133,7 @@ describe('finalizeTraceFile', () => { finalizeTraceFile(events as any, outputPath); - expect(fs.existsSync(outputPath)).toBe(true); - const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); - expect(content.traceEvents).toHaveLength(3); // preamble (start + complete) + end complete + expect(fs.existsSync(outputPath)).toBe(false); // No file created for empty events }); it('should sort events by timestamp', () => { @@ -154,6 +151,46 @@ describe('finalizeTraceFile', () => { .map((e: any) => e.name); expect(eventNames).toStrictEqual(['event1', 'event2']); }); + + it('should use configurable margins', () => { + const events: TraceEvent[] = [{ name: 'event1', ts: 1000, ph: 'I' }]; + const outputPath = '/tmp/custom-margin-trace.json'; + + finalizeTraceFile( + events as any, + outputPath, + {}, + { marginMs: 500, marginDurMs: 10 }, + ); + + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + expect(content.traceEvents).toHaveLength(4); // start tracing + start margin + event + end margin + + // Check start margin timestamp and duration + const startMargin = content.traceEvents.find( + (e: any) => e.name === '[trace padding start]', + ); + expect(startMargin.ts).toBe(500); // 1000 - 500 + expect(startMargin.dur).toBe(10); + + // Check end margin timestamp and duration + const endMargin = content.traceEvents.find( + (e: any) => e.name === '[trace padding end]', + ); + expect(endMargin.ts).toBe(1500); // 1000 + 500 + expect(endMargin.dur).toBe(10); + }); + + it('should use deterministic startTime', () => { + const events: TraceEvent[] = [{ name: 'event1', ts: 1000, ph: 'I' }]; + const outputPath = '/tmp/deterministic-trace.json'; + const fixedTime = '2023-01-15T10:30:00.000Z'; + + finalizeTraceFile(events as any, outputPath, {}, { startTime: fixedTime }); + + const content = JSON.parse(fs.readFileSync(outputPath, 'utf8')); + expect(content.metadata.startTime).toBe(fixedTime); + }); }); describe('TraceFileSink', () => { @@ -167,13 +204,13 @@ describe('TraceFileSink', () => { }); it('should create trace file sink with default options', () => { - const sink = new TraceFileSink({ filename: 'test' }); + const sink = new FileSinkJsonTrace({ filename: 'test' }); expect(sink.getFilePathForExt('json')).toBe('test.json'); expect(sink.getFilePathForExt('jsonl')).toBe('test.jsonl'); }); it('should create trace file sink with custom directory', () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'test', directory: '/tmp/custom', }); @@ -182,12 +219,10 @@ describe('TraceFileSink', () => { }); it('should handle file operations with trace events', () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'trace-test', directory: '/tmp', }); - sink.open(); - const event1: InstantEvent = { name: 'mark1', ts: 100, ph: 'I' }; const event2: InstantEvent = { name: 'mark2', ts: 200, ph: 'I' }; sink.write(event1); @@ -202,11 +237,10 @@ describe('TraceFileSink', () => { }); it('should create trace file on finalize', () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'finalize-test', directory: '/tmp', }); - sink.open(); const event: InstantEvent = { name: 'test-event', ts: 150, ph: 'I' }; sink.write(event); @@ -224,12 +258,11 @@ describe('TraceFileSink', () => { it('should handle metadata in finalize', () => { const metadata = { customField: 'value', version: '1.0' }; - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'metadata-test', directory: '/tmp', metadata, }); - sink.open(); sink.write({ name: 'event', ts: 100, ph: 'I' }); sink.finalize(); @@ -240,17 +273,43 @@ describe('TraceFileSink', () => { expect(content.metadata.version).toBe('1.0'); }); + it('should use configurable options in TraceFileSink', () => { + const sink = new FileSinkJsonTrace({ + filename: 'options-test', + directory: '/tmp', + marginMs: 200, + marginDurMs: 5, + startTime: '2023-12-25T12:00:00.000Z', + }); + sink.write({ name: 'event', ts: 1000, ph: 'I' }); + sink.finalize(); + + const content = JSON.parse( + fs.readFileSync('/tmp/options-test.json', 'utf8'), + ); + expect(content.metadata.startTime).toBe('2023-12-25T12:00:00.000Z'); + + const startMargin = content.traceEvents.find( + (e: any) => e.name === '[trace padding start]', + ); + expect(startMargin.ts).toBe(800); // 1000 - 200 + expect(startMargin.dur).toBe(5); + + const endMargin = content.traceEvents.find( + (e: any) => e.name === '[trace padding end]', + ); + expect(endMargin.ts).toBe(1200); // 1000 + 200 + expect(endMargin.dur).toBe(5); + }); + it('should do nothing on finalize when no events written', () => { - const sink = new TraceFileSink({ + const sink = new FileSinkJsonTrace({ filename: 'empty-test', directory: '/tmp', }); - sink.open(); sink.finalize(); - expect(fs.existsSync('/tmp/empty-test.json')).toBe(true); - const content = JSON.parse(fs.readFileSync('/tmp/empty-test.json', 'utf8')); - expect(content.traceEvents).toHaveLength(3); // preamble (start + complete) + end complete + expect(fs.existsSync('/tmp/empty-test.json')).toBe(false); // No file created for empty events }); }); diff --git a/packages/utils/src/lib/file-sink-jsonl.int.test.ts b/packages/utils/src/lib/file-sink-jsonl.int.test.ts index e0f57bbaa..c0bae5503 100644 --- a/packages/utils/src/lib/file-sink-jsonl.int.test.ts +++ b/packages/utils/src/lib/file-sink-jsonl.int.test.ts @@ -3,9 +3,9 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { teardownTestFolder } from '@code-pushup/test-utils'; -import { JsonlFileSink, recoverJsonlFile } from './file-sink-jsonl.js'; +import { JsonlFile, recoverJsonlFile } from './file-sink-jsonl.js'; -describe('JsonlFileSink integration', () => { +describe('JsonlFile integration', () => { const baseDir = path.join(os.tmpdir(), 'file-sink-json-int-tests'); const testFile = path.join(baseDir, 'test-data.jsonl'); @@ -33,7 +33,7 @@ describe('JsonlFileSink integration', () => { ]; it('should write and read JSONL files', async () => { - const sink = new JsonlFileSink({ filePath: testFile }); + const sink = new JsonlFile({ filePath: testFile }); // Open and write data sink.open(); @@ -91,7 +91,7 @@ describe('JsonlFileSink integration', () => { }); it('should recover data using JsonlFileSink.recover()', async () => { - const sink = new JsonlFileSink({ filePath: testFile }); + const sink = new JsonlFile({ filePath: testFile }); sink.open(); testData.forEach(item => sink.write(item)); sink.close(); diff --git a/packages/utils/src/lib/file-sink-jsonl.ts b/packages/utils/src/lib/file-sink-jsonl.ts index 646cd82b1..c3e9f2a83 100644 --- a/packages/utils/src/lib/file-sink-jsonl.ts +++ b/packages/utils/src/lib/file-sink-jsonl.ts @@ -1,60 +1,173 @@ import * as fs from 'node:fs'; -import { - type FileOutput, - FileSink, - type FileSinkOptions, - stringDecode, - stringEncode, - stringRecover, -} from './file-sink-text.js'; -import type { RecoverOptions, RecoverResult } from './sink-source.types.js'; +import { TextFileSink } from './file-sink-text.js'; +import type { RecoverOptions, RecoverResult } from './sink-source.type.js'; +/** + * JSONL encoding functions - single source of truth for JSONL format. + */ export const jsonlEncode = < T extends Record = Record, >( input: T, -): FileOutput => JSON.stringify(input); +): string => JSON.stringify(input); export const jsonlDecode = < T extends Record = Record, >( - output: FileOutput, -): T => JSON.parse(stringDecode(output)) as T; + raw: string, +): T => JSON.parse(raw) as T; export function recoverJsonlFile< T extends Record = Record, >(filePath: string, opts: RecoverOptions = {}): RecoverResult { - return stringRecover(filePath, jsonlDecode, opts); + const records: T[] = []; + const errors: { lineNo: number; line: string; error: Error }[] = []; + let partialTail: string | null = null; + + try { + const content = fs.readFileSync(filePath, 'utf8'); + const lines = content.split('\n'); + let lineNo = 0; + + for (const line of lines) { + lineNo++; + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const record = jsonlDecode(trimmedLine); + records.push(record); + } catch (error) { + const info = { lineNo, line, error: error as Error }; + errors.push(info); + + if (opts.keepInvalid) { + records.push({ __invalid: true, lineNo, line } as any); + } + + partialTail = line; + } + + // Optional: perfect tail detection for empty lines at EOF + if (trimmedLine === '' && lineNo === lines.length) { + partialTail = line; + } + } + } catch { + return { records: [], errors: [], partialTail: null }; + } + + return { records, errors, partialTail }; } -export class JsonlFileSink< +export type JsonlFileOptions> = { + filePath: string; + recover?: () => RecoverResult; + finalize?: () => void; +}; + +/** + * JSONL writer using composition: Transport + Encoding + Recovery policy. + * Writes are append-only. + * + * JsonlFile opens the underlying file lazily on first write and keeps it open + * until close() or finalize() is called. + * + * Design rules: + * - "Extend types only when substitutable" + * - "Reuse behavior via composition" + * - "Transport ≠ format ≠ recovery" + */ +export class JsonlFile< T extends Record = Record, -> extends FileSink { - constructor(options: FileSinkOptions) { - const { filePath, ...fileOptions } = options; - super({ - ...fileOptions, - filePath, - recover: () => recoverJsonlFile(filePath), - finalize: () => { - // No additional finalization needed for JSONL files - }, - }); +> { + private file: TextFileSink; + + constructor(options: JsonlFileOptions) { + const { filePath } = options; + this.file = new TextFileSink(filePath); + + // Recovery policy - JSONL-specific, customizable + this.recover = options.recover ?? (() => recoverJsonlFile(filePath)); + + // Finalization policy - defaults to close() for cleanup + this.finalize = options.finalize ?? (() => this.close()); } - override encode(input: T): FileOutput { - return stringEncode(jsonlEncode(input)); + /** + * Encode record to JSONL format. + */ + encode(record: T): string { + return jsonlEncode(record) + '\n'; + } + + /** + * Decode JSONL string to record. + */ + decode(jsonlString: string): T { + return jsonlDecode(jsonlString); + } + + /** + * Open file for writing (no-op since TextFileSink opens lazily). + */ + open(): void { + // TextFileSink opens lazily on first write, so no-op here + } + + /** + * Write record in JSONL format (append-only). + */ + write(record: T): void { + this.file.append(jsonlEncode(record) + '\n'); + } + + /** + * Read all records as parsed array (strict - throws on invalid JSON). + */ + readAll(): T[] { + return this.file + .readAll() + .split('\n') + .filter(Boolean) + .map(line => jsonlDecode(line)); + } + + /** + * Recover records with error handling (tolerant parsing). + * Handles invalid records gracefully, returns errors alongside valid data. + */ + recover: () => RecoverResult; + + /** + * Finalization - defaults to close() for cleanup. + */ + finalize: () => void; + + /** + * Get file path. + */ + getPath(): string { + return this.file.getPath(); } - override decode(output: FileOutput): T { - return jsonlDecode(stringDecode(output)); + /** + * Close file. + */ + close(): void { + this.file.close(); } - override repack(outputPath?: string): void { + /** + * Repack file with clean JSONL formatting. + */ + repack(outputPath?: string): void { const { records } = this.recover(); fs.writeFileSync( - outputPath ?? this.getFilePath(), - records.map(this.encode).join(''), + outputPath ?? this.getPath(), + records.map(jsonlEncode).join('\n') + '\n', ); } } diff --git a/packages/utils/src/lib/file-sink-jsonl.unit.test.ts b/packages/utils/src/lib/file-sink-jsonl.unit.test.ts index 75f981cb0..7d775820f 100644 --- a/packages/utils/src/lib/file-sink-jsonl.unit.test.ts +++ b/packages/utils/src/lib/file-sink-jsonl.unit.test.ts @@ -3,7 +3,7 @@ import * as fs from 'node:fs'; import { beforeEach, describe, expect, it } from 'vitest'; import { MEMFS_VOLUME } from '@code-pushup/test-utils'; import { - JsonlFileSink, + JsonlFile, jsonlDecode, jsonlEncode, recoverJsonlFile, @@ -150,7 +150,7 @@ describe('recoverJsonlFile', () => { }); }); -describe('JsonlFileSink', () => { +describe('JsonlFile', () => { beforeEach(() => { vol.fromJSON( { @@ -163,7 +163,7 @@ describe('JsonlFileSink', () => { type JsonObj = { key: string; number: number }; it('should encode objects as JSON', () => { - const sink = new JsonlFileSink({ + const sink = new JsonlFile({ filePath: '/tmp/jsonl-test.jsonl', }); const obj = { key: 'value', number: 42 }; @@ -171,7 +171,7 @@ describe('JsonlFileSink', () => { }); it('should decode JSON strings to objects', () => { - const sink = new JsonlFileSink({ + const sink = new JsonlFile({ filePath: '/tmp/jsonl-test.jsonl', }); const obj = { key: 'value', number: 42 }; @@ -181,8 +181,7 @@ describe('JsonlFileSink', () => { it('should handle file operations with JSONL format', () => { const filePath = '/tmp/jsonl-file-ops-test.jsonl'; - const sink = new JsonlFileSink({ filePath }); - sink.open(); + const sink = new JsonlFile({ filePath }); const obj1 = { key: 'value', number: 42 }; const obj2 = { key: 'value', number: 42 }; @@ -196,7 +195,7 @@ describe('JsonlFileSink', () => { it('repack() should recover records and write them to output path', () => { const filePath = '/tmp/jsonl-repack-test.jsonl'; - const sink = new JsonlFileSink({ filePath }); + const sink = new JsonlFile({ filePath }); const records = [ { key: 'value', number: 42 }, { key: 'value', number: 42 }, @@ -215,7 +214,7 @@ describe('JsonlFileSink', () => { it('repack() should accept output path', () => { const filePath = '/tmp/jsonl-repack-test.jsonl'; - const sink = new JsonlFileSink({ filePath }); + const sink = new JsonlFile({ filePath }); const records = [ { key: 'value', number: 42 }, { key: 'value', number: 42 }, @@ -234,7 +233,7 @@ describe('JsonlFileSink', () => { }); it('should do nothing on finalize()', () => { - const sink = new JsonlFileSink({ + const sink = new JsonlFile({ filePath: '/tmp/jsonl-finalize-test.jsonl', }); expect(() => sink.finalize()).not.toThrow(); diff --git a/packages/utils/src/lib/file-sink-text.ts b/packages/utils/src/lib/file-sink-text.ts index 050188e58..80c617be7 100644 --- a/packages/utils/src/lib/file-sink-text.ts +++ b/packages/utils/src/lib/file-sink-text.ts @@ -1,36 +1,93 @@ +/** + * Simple Text File Sink + * + * Basic file operations for text files. Used as the foundation for format-specific writers. + * If you need JSONL files, use JsonlFile from file-sink-jsonl.ts instead. + */ import * as fs from 'node:fs'; -import { existsSync, mkdirSync } from 'node:fs'; -import path from 'node:path'; -import { PROFILER_FILE_BASE_NAME, PROFILER_OUT_DIR } from './profiler'; -import type { - RecoverOptions, - RecoverResult, - Recoverable, - Sink, -} from './sink-source.types.js'; - -export const stringDecode = (output: O): I => { - const str = Buffer.isBuffer(output) - ? output.toString('utf8') - : String(output); - return str as unknown as I; +import * as path from 'node:path'; +import type { RecoverOptions, RecoverResult } from './sink-source.type.js'; + +/** + * Simple text file sink - reusable for basic file operations. + * One responsibility: append text, read all text, get path. + */ +export class TextFileSink { + #fd: number | null = null; + + constructor(private filePath: string) {} + + /** + * Append text to file (append-only). + */ + append(text: string): void { + // Lazy open on first write + if (this.#fd === null) { + const dir = path.dirname(this.filePath); + fs.mkdirSync(dir, { recursive: true }); + this.#fd = fs.openSync(this.filePath, 'a'); + } + fs.writeSync(this.#fd, text); + } + + /** + * Read entire file as string. + */ + readAll(): string { + try { + return fs.readFileSync(this.filePath, 'utf8'); + } catch { + return ''; + } + } + + /** + * Get file path. + */ + getPath(): string { + return this.filePath; + } + + /** + * Close file descriptor. + */ + close(): void { + if (this.#fd !== null) { + fs.closeSync(this.#fd); + this.#fd = null; + } + } +} + +/** + * String encoding functions - single source of truth for string format. + */ +export const stringEncode = (input: unknown): string => { + if (typeof input === 'string') { + return `${input}\n`; + } + return `${JSON.stringify(input)}\n`; }; -export const stringEncode = (input: I): O => - `${typeof input === 'string' ? input : JSON.stringify(input)}\n` as O; +export const stringDecode = (input: string | Buffer): string => { + if (Buffer.isBuffer(input)) { + return input.toString('utf8'); + } + return input; +}; -export const stringRecover = function ( +export function stringRecover( filePath: string, - decode: (output: O) => I, + decodeFn: (line: string) => T, opts: RecoverOptions = {}, -): RecoverResult { - const records: I[] = []; +): RecoverResult { + const records: T[] = []; const errors: { lineNo: number; line: string; error: Error }[] = []; let partialTail: string | null = null; try { const content = fs.readFileSync(filePath, 'utf8'); - const lines = content.trim().split('\n'); + const lines = content.split('\n'); let lineNo = 0; for (const line of lines) { @@ -41,7 +98,7 @@ export const stringRecover = function ( } try { - const record = decode(trimmedLine as O); + const record = decodeFn(trimmedLine); records.push(record); } catch (error) { const info = { lineNo, line, error: error as Error }; @@ -53,132 +110,148 @@ export const stringRecover = function ( partialTail = line; } + + // Optional: perfect tail detection for empty lines at EOF + if (trimmedLine === '' && lineNo === lines.length) { + partialTail = line; + } } } catch { return { records: [], errors: [], partialTail: null }; } return { records, errors, partialTail }; -}; - -export type FileNameOptions = { - fileBaseName: string; - outDir: string; - fileName?: string; -}; - -export function getFilenameParts(options: FileNameOptions): { - outDir: string; - fileName: string; -} { - const { fileName, fileBaseName, outDir } = options; - - if (fileName) { - return { - outDir, - fileName, - }; - } - - const baseName = fileBaseName; - const DATE_LENGTH = 10; - const TIME_SEGMENTS = 3; - const COLON_LENGTH = 1; - const TOTAL_TIME_LENGTH = - TIME_SEGMENTS * 2 + (TIME_SEGMENTS - 1) * COLON_LENGTH; // HH:MM:SS = 8 chars - const id = new Date() - .toISOString() - .slice(0, DATE_LENGTH + TOTAL_TIME_LENGTH) - .replace(/:/g, '-'); - - return { - outDir, - fileName: `${baseName}.${id}`, - }; } -export type FileSinkOptions = { +export type FileSinkOptions = { filePath: string; - recover?: () => RecoverResult; + recover?: () => RecoverResult; finalize?: () => void; }; -export type FileInput = Buffer | string; -export type FileOutput = Buffer | string; +/** + * String file sink using composition: Transport + Encoding + Recovery policy. + * Writes are append-only. + * + * FileSink opens the underlying file lazily on first write and keeps it open + * until close() or finalize() is called. + * + * Design rules: + * - "Extend types only when substitutable" + * - "Reuse behavior via composition" + * - "Transport ≠ format ≠ recovery" + */ +export class FileSink { + private file: TextFileSink; + private isOpen = false; + private fd: number | null = null; -export class FileSink - implements Sink, Recoverable -{ - #fd: number | null = null; - options: FileSinkOptions; + constructor(public options: FileSinkOptions) { + const { filePath } = options; + this.file = new TextFileSink(filePath); - constructor(options: FileSinkOptions) { - this.options = options; - } + // Recovery policy - string-specific, customizable + this.recover = + options.recover ?? + (() => stringRecover(filePath, (line: string) => line as T)); - isClosed(): boolean { - return this.#fd == null; + // Finalization policy - defaults to close() for cleanup + this.finalize = options.finalize ?? (() => this.close()); } - encode(input: I): O { - return stringEncode(input as any); + /** + * Encode input to string format. + */ + encode(input: T): string { + return stringEncode(input); } - decode(output: O): I { - return stringDecode(output as any); + /** + * Decode string to output type. + */ + decode(output: string | Buffer): T { + const str = stringDecode(output); + return str as T; } + + /** + * Get file path. + */ getFilePath(): string { - return this.options.filePath; + return this.file.getPath(); } - open(withRepack: boolean = false): void { - const dir = path.dirname(this.options.filePath); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }); - } + /** + * Open file for writing (creates directory if needed). + */ + open(withRepack?: boolean): void { + if (this.isOpen) return; + + const dir = path.dirname(this.file.getPath()); + fs.mkdirSync(dir, { recursive: true }); + if (withRepack) { - this.repack(this.options.filePath); + this.repack(this.file.getPath()); } - this.#fd = fs.openSync(this.options.filePath, 'a'); - } - close(): void { - if (this.#fd == null) { - return; - } - fs.closeSync(this.#fd); - this.#fd = null; + this.fd = fs.openSync(this.file.getPath(), 'a'); + this.isOpen = true; } - write(input: I): void { - if (this.#fd == null) { - return; - } // Silently ignore if not open - const encoded = this.encode(input); + /** + * Write input to file (append-only). + */ + write(input: T): void { + if (!this.isOpen) return; + try { - fs.writeSync(this.#fd, encoded as any); + const encoded = this.encode(input); + fs.writeSync(this.fd!, encoded); } catch { - // Silently ignore write errors (e.g., EBADF in test environments with mocked fs) + // Silently ignore write errors } } - recover(): RecoverResult { - const dir = path.dirname(this.options.filePath); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true }); + /** + * Close file descriptor. + */ + close(): void { + if (this.fd !== null) { + fs.closeSync(this.fd); + this.fd = null; } - return this.options.recover!() as RecoverResult; + this.isOpen = false; } + /** + * Check if sink is closed. + */ + isClosed(): boolean { + return !this.isOpen; + } + + /** + * Recover records with error handling (tolerant parsing). + * Handles invalid records gracefully, returns errors alongside valid data. + */ + recover: () => RecoverResult; + + /** + * Repack file with clean formatting. + */ repack(outputPath?: string): void { const { records } = this.recover(); + const targetPath = outputPath ?? this.getFilePath(); + const dir = path.dirname(targetPath); + fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync( - outputPath ?? this.getFilePath(), - records.map(this.encode).join('\n'), + targetPath, + records.map(record => this.encode(record)).join(''), ); } - finalize(): void { - this.options.finalize!(); - } + /** + * Finalization - defaults to close() for cleanup. + */ + finalize: () => void; } diff --git a/packages/utils/src/lib/file-sink-text.unit.test.ts b/packages/utils/src/lib/file-sink-text.unit.test.ts index 33cc9ad0e..eb1b17749 100644 --- a/packages/utils/src/lib/file-sink-text.unit.test.ts +++ b/packages/utils/src/lib/file-sink-text.unit.test.ts @@ -262,7 +262,7 @@ describe('FileSink', () => { sink.repack(); expect(mockRecover).toHaveBeenCalled(); - expect(fs.readFileSync(filePath, 'utf8')).toBe('record1\n\nrecord2\n'); + expect(fs.readFileSync(filePath, 'utf8')).toBe('record1\nrecord2\n'); }); it('repack() should accept output path', () => { @@ -276,7 +276,7 @@ describe('FileSink', () => { const outputPath = '/tmp/repack-output.txt'; sink.repack(outputPath); expect(mockRecover).toHaveBeenCalled(); - expect(fs.readFileSync(outputPath, 'utf8')).toBe('record1\n\nrecord2\n'); + expect(fs.readFileSync(outputPath, 'utf8')).toBe('record1\nrecord2\n'); }); it('finalize() should call the finalize function from options', () => { diff --git a/packages/utils/src/lib/trace-file-utils.ts b/packages/utils/src/lib/trace-file-utils.ts index 2a2f3eb30..09e028881 100644 --- a/packages/utils/src/lib/trace-file-utils.ts +++ b/packages/utils/src/lib/trace-file-utils.ts @@ -1,7 +1,7 @@ -import os from 'node:os'; import type { PerformanceMark, PerformanceMeasure } from 'node:perf_hooks'; import { threadId } from 'node:worker_threads'; import { defaultClock } from './clock-epoch.js'; +import { jsonlDecode, jsonlEncode } from './file-sink-jsonl.js'; import type { BeginEvent, CompleteEvent, @@ -13,7 +13,9 @@ import type { SpanEventArgs, TraceEvent, TraceEventContainer, + TraceEventRaw, } from './trace-file.type.js'; +import type { UserTimingDetail } from './user-timing-extensibility-api.type.js'; /** Global counter for generating unique span IDs within a trace */ // eslint-disable-next-line functional/no-let @@ -228,7 +230,7 @@ export const markToInstantEvent = ( ...opt, name: opt?.name ?? entry.name, ts: defaultClock.fromEntry(entry), - args: entry.detail ? { detail: entry.detail } : undefined, + args: entry.detail ? { data: { detail: entry.detail } } : undefined, }); /** @@ -249,6 +251,19 @@ export const measureToSpanEvents = ( args: entry.detail ? { data: { detail: entry.detail } } : undefined, }); +export function getTraceMetadata( + startDate?: Date, + metadata?: Record, +) { + return { + source: 'DevTools', + startTime: startDate?.toISOString() ?? new Date().toISOString(), + hardwareConcurrency: 1, + dataOrigin: 'TraceEvents', + ...metadata, + }; +} + /** * Creates a complete trace file container with metadata. * @param opt - Trace file configuration @@ -263,6 +278,65 @@ export const getTraceFile = (opt: { metadata: { source: 'Node.js UserTiming', startTime: opt.startTime ?? new Date().toISOString(), - hardwareConcurrency: os.cpus().length, + hardwareConcurrency: 1, }, }); + +function processDetail( + target: T, + processor: (detail: string | object) => string | object, +): T { + if ( + target.detail != null && + (typeof target.detail === 'string' || typeof target.detail === 'object') + ) { + return { ...target, detail: processor(target.detail) }; + } + return target; +} + +export function decodeDetail(target: { detail: string }): UserTimingDetail { + return processDetail(target, detail => + typeof detail === 'string' ? jsonlDecode(detail) : detail, + ) as UserTimingDetail; +} + +export function encodeDetail(target: UserTimingDetail): UserTimingDetail { + return processDetail(target, detail => + typeof detail === 'object' + ? jsonlEncode(detail as UserTimingDetail) + : detail, + ); +} + +export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { + if (!args) return rest as TraceEvent; + + const processedArgs = decodeDetail(args as { detail: string }); + if ('data' in args && args.data && typeof args.data === 'object') { + return { + ...rest, + args: { + ...processedArgs, + data: decodeDetail(args.data as { detail: string }), + }, + } as TraceEvent; + } + return { ...rest, args: processedArgs } as TraceEvent; +} + +export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { + if (!args) return rest as TraceEventRaw; + + const processedArgs = encodeDetail(args); + if ('data' in args && args.data && typeof args.data === 'object') { + return { + ...rest, + args: { + ...processedArgs, + data: encodeDetail(args.data as UserTimingDetail), + }, + } as TraceEventRaw; + } + return { ...rest, args: processedArgs } as TraceEventRaw; +} diff --git a/testing/test-utils/src/lib/utils/perf-hooks.mock.ts b/testing/test-utils/src/lib/utils/perf-hooks.mock.ts index b22e88bd5..d7a8b3ab1 100644 --- a/testing/test-utils/src/lib/utils/perf-hooks.mock.ts +++ b/testing/test-utils/src/lib/utils/perf-hooks.mock.ts @@ -33,27 +33,36 @@ export const createPerformanceMock = (timeOrigin = 500_000) => ({ now: vi.fn(() => nowMs), - mark: vi.fn((name: string) => { + mark: vi.fn((name: string, options?: { detail?: unknown }) => { entries.push({ name, entryType: 'mark', startTime: nowMs, duration: 0, + detail: options?.detail, } as PerformanceEntry); MockPerformanceObserver.globalEntries = entries; }), - measure: vi.fn((name: string, startMark?: string, endMark?: string) => { - const entry = { - name, - entryType: 'measure', - startTime: nowMs, - duration: nowMs, - } as PerformanceEntry; - entries.push(entry); - MockPerformanceObserver.globalEntries = entries; - triggerObservers([entry]); - }), + measure: vi.fn( + ( + name: string, + startMark?: string, + endMark?: string, + options?: { detail?: unknown }, + ) => { + const entry = { + name, + entryType: 'measure', + startTime: nowMs, + duration: nowMs, + detail: options?.detail, + } as PerformanceEntry; + entries.push(entry); + MockPerformanceObserver.globalEntries = entries; + triggerObservers([entry]); + }, + ), getEntries: vi.fn(() => entries.slice()), From 67e004c713a186d39e9636cfa2b8f81829228079 Mon Sep 17 00:00:00 2001 From: Michael Hladky Date: Wed, 14 Jan 2026 22:53:50 +0100 Subject: [PATCH 7/7] refactor: fix lint --- packages/utils/src/lib/file-sink-json-trace.int.test.ts | 2 +- packages/utils/src/lib/file-sink-json-trace.ts | 6 ++++-- packages/utils/src/lib/file-sink-json-trace.unit.test.ts | 5 ++++- packages/utils/src/lib/file-sink-jsonl.ts | 6 +++--- packages/utils/src/lib/file-sink-text.ts | 8 ++++++-- packages/utils/src/lib/trace-file-utils.ts | 8 ++++++-- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/packages/utils/src/lib/file-sink-json-trace.int.test.ts b/packages/utils/src/lib/file-sink-json-trace.int.test.ts index b9cdb7f2e..de0bc213b 100644 --- a/packages/utils/src/lib/file-sink-json-trace.int.test.ts +++ b/packages/utils/src/lib/file-sink-json-trace.int.test.ts @@ -3,7 +3,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { teardownTestFolder } from '@code-pushup/test-utils'; -import { FileSinkJsonTrace } from './file-sink-json-trace'; +import { FileSinkJsonTrace } from './file-sink-json-trace.js'; import type { CompleteEvent, TraceEvent } from './trace-file.type'; describe('TraceFileSink integration', () => { diff --git a/packages/utils/src/lib/file-sink-json-trace.ts b/packages/utils/src/lib/file-sink-json-trace.ts index 6e201f133..880a1436b 100644 --- a/packages/utils/src/lib/file-sink-json-trace.ts +++ b/packages/utils/src/lib/file-sink-json-trace.ts @@ -37,7 +37,7 @@ export function finalizeTraceFile( events.sort((a, b) => a.ts - b.ts); const fallbackTs = performance.now(); const firstTs = events.length > 0 ? events[0].ts : fallbackTs; - const lastTs = events.length > 0 ? events[events.length - 1].ts : fallbackTs; + const lastTs = events.length > 0 ? events.at(-1).ts : fallbackTs; const marginMs = options?.marginMs ?? TRACE_MARGIN_MS; const marginDurMs = options?.marginDurMs ?? TRACE_MARGIN_DURATION_MS; @@ -182,7 +182,9 @@ export class FileSinkJsonTrace { } finalize(): void { - if (this.#finalized) return; + if (this.#finalized) { + return; + } this.#finalized = true; this.sink.finalize(); } diff --git a/packages/utils/src/lib/file-sink-json-trace.unit.test.ts b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts index 30911b558..1b7ae244d 100644 --- a/packages/utils/src/lib/file-sink-json-trace.unit.test.ts +++ b/packages/utils/src/lib/file-sink-json-trace.unit.test.ts @@ -2,7 +2,10 @@ import { vol } from 'memfs'; import * as fs from 'node:fs'; import { beforeEach, describe, expect, it } from 'vitest'; import { MEMFS_VOLUME } from '@code-pushup/test-utils'; -import { FileSinkJsonTrace, finalizeTraceFile } from './file-sink-json-trace'; +import { + FileSinkJsonTrace, + finalizeTraceFile, +} from './file-sink-json-trace.js'; import { decodeTraceEvent, encodeTraceEvent, diff --git a/packages/utils/src/lib/file-sink-jsonl.ts b/packages/utils/src/lib/file-sink-jsonl.ts index c3e9f2a83..f2eeebd88 100644 --- a/packages/utils/src/lib/file-sink-jsonl.ts +++ b/packages/utils/src/lib/file-sink-jsonl.ts @@ -100,7 +100,7 @@ export class JsonlFile< * Encode record to JSONL format. */ encode(record: T): string { - return jsonlEncode(record) + '\n'; + return `${jsonlEncode(record)}\n`; } /** @@ -121,7 +121,7 @@ export class JsonlFile< * Write record in JSONL format (append-only). */ write(record: T): void { - this.file.append(jsonlEncode(record) + '\n'); + this.file.append(`${jsonlEncode(record)}\n`); } /** @@ -167,7 +167,7 @@ export class JsonlFile< const { records } = this.recover(); fs.writeFileSync( outputPath ?? this.getPath(), - records.map(jsonlEncode).join('\n') + '\n', + `${records.map(jsonlEncode).join('\n')}\n`, ); } } diff --git a/packages/utils/src/lib/file-sink-text.ts b/packages/utils/src/lib/file-sink-text.ts index 80c617be7..273608323 100644 --- a/packages/utils/src/lib/file-sink-text.ts +++ b/packages/utils/src/lib/file-sink-text.ts @@ -185,7 +185,9 @@ export class FileSink { * Open file for writing (creates directory if needed). */ open(withRepack?: boolean): void { - if (this.isOpen) return; + if (this.isOpen) { + return; + } const dir = path.dirname(this.file.getPath()); fs.mkdirSync(dir, { recursive: true }); @@ -202,7 +204,9 @@ export class FileSink { * Write input to file (append-only). */ write(input: T): void { - if (!this.isOpen) return; + if (!this.isOpen) { + return; + } try { const encoded = this.encode(input); diff --git a/packages/utils/src/lib/trace-file-utils.ts b/packages/utils/src/lib/trace-file-utils.ts index 09e028881..1f08054e8 100644 --- a/packages/utils/src/lib/trace-file-utils.ts +++ b/packages/utils/src/lib/trace-file-utils.ts @@ -310,7 +310,9 @@ export function encodeDetail(target: UserTimingDetail): UserTimingDetail { } export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { - if (!args) return rest as TraceEvent; + if (!args) { + return rest as TraceEvent; + } const processedArgs = decodeDetail(args as { detail: string }); if ('data' in args && args.data && typeof args.data === 'object') { @@ -326,7 +328,9 @@ export function decodeTraceEvent({ args, ...rest }: TraceEventRaw): TraceEvent { } export function encodeTraceEvent({ args, ...rest }: TraceEvent): TraceEventRaw { - if (!args) return rest as TraceEventRaw; + if (!args) { + return rest as TraceEventRaw; + } const processedArgs = encodeDetail(args); if ('data' in args && args.data && typeof args.data === 'object') {