From 4fd7143c611b2b7f5249ca653f05a440231d4514 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Thu, 6 Mar 2025 12:30:54 +0000 Subject: [PATCH 01/22] parseFileHeader not async --- src/sparse.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index 8d34dfc..1697ec4 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -103,18 +103,18 @@ export class Sparse { * @returns {Promise} */ export async function from(blob) { - const header = await parseFileHeader(blob); + const header = parseFileHeader(await blob.slice(0, FILE_HEADER_SIZE).arrayBuffer()); if (!header) return null; return new Sparse(blob, header); } /** - * @param {Blob} blob - * @returns {Promise} + * @param {Uint8Array} buffer + * @returns {Header|null} */ -export async function parseFileHeader(blob) { - const view = new DataView(await blob.slice(0, FILE_HEADER_SIZE).arrayBuffer()); +export function parseFileHeader(buffer) { + const view = new DataView(buffer); const magic = view.getUint32(0, true); if (magic !== FILE_MAGIC) { return null; From 44ef56a9547ca057e1e202b6fcbbf17c5d566e94 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Thu, 6 Mar 2025 12:31:46 +0000 Subject: [PATCH 02/22] better API --- src/sparse.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index 1697ec4..ecf0169 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -117,17 +117,16 @@ export function parseFileHeader(buffer) { const view = new DataView(buffer); const magic = view.getUint32(0, true); if (magic !== FILE_MAGIC) { + // Not a sparse file. return null; } const fileHeaderSize = view.getUint16(8, true); const chunkHeaderSize = view.getUint16(10, true); if (fileHeaderSize !== FILE_HEADER_SIZE) { - console.error(`The file header size was expected to be 28, but is ${fileHeaderSize}.`); - return null; + throw `Sparse - The file header size was expected to be 28, but is ${fileHeaderSize}.`; } if (chunkHeaderSize !== CHUNK_HEADER_SIZE) { - console.error(`The chunk header size was expected to be 12, but is ${chunkHeaderSize}.`); - return null; + throw `Sparse - The chunk header size was expected to be 12, but is ${chunkHeaderSize}.`; } return { magic, From ee6e6fb80aac65acf2de6497411e712cbd6ad02c Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Thu, 6 Mar 2025 17:07:04 +0000 Subject: [PATCH 03/22] ReadableStream --- scripts/simg2img.js | 19 ++++-- src/sparse.js | 158 ++++++++++++++++++++++++++------------------ src/sparse.spec.js | 30 ++------- 3 files changed, 112 insertions(+), 95 deletions(-) diff --git a/scripts/simg2img.js b/scripts/simg2img.js index 1782370..a405409 100755 --- a/scripts/simg2img.js +++ b/scripts/simg2img.js @@ -5,19 +5,28 @@ export async function simg2img(inputPath, outputPath) { const sparseImage = Bun.file(inputPath); const outputImage = Bun.file(outputPath); - const sparse = await Sparse.from(sparseImage); - if (!sparse) throw "Failed to parse sparse file"; + const result = await Sparse.from(sparseImage.stream()); + if (!result) throw "Failed to parse sparse file"; // FIXME: write out a "sparse" file? not supported by Bun const writer = outputImage.writer({ highWaterMark: 4 * 1024 * 1024 }); - for await (const [_, chunk, size] of sparse.read()) { - if (chunk) { - writer.write(await chunk.arrayBuffer()); + + const stream = Sparse.read(...result); + const reader = stream.getReader(); + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + const [_, data, size ] = value; + if (data) { + writer.write(data); } else { writer.write(new Uint8Array(size).buffer); } } + writer.end(); + reader.releaseLock(); } if (import.meta.main) { diff --git a/src/sparse.js b/src/sparse.js index ecf0169..b74bbdb 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -1,3 +1,5 @@ +import { concatUint8Array } from "./utils"; + const FILE_MAGIC = 0xed26ff3a; export const FILE_HEADER_SIZE = 28; const CHUNK_HEADER_SIZE = 12; @@ -11,7 +13,7 @@ const ChunkType = { /** - * @typedef {object} Header + * @typedef {object} SparseHeader * @property {number} magic * @property {number} majorVersion * @property {number} minorVersion @@ -25,93 +27,117 @@ const ChunkType = { /** - * @typedef {object} Chunk + * @typedef {object} SparseChunk * @property {number} type * @property {number} blocks - * @property {Blob} data + * @property {Uint8Array} data + */ + + +/** + * @param {ReadableStream} stream + * @returns {Promise<[SparseHeader, ReadableStream] | null>} */ +export async function from(stream) { + const reader = stream.getReader(); + let buffer = new Uint8Array(0); + const readUntil = async (byteLength) => { + while (buffer.byteLength < byteLength) { + const { value, done } = await reader.read(); + if (done) throw new Error("Unexpected end of stream"); + buffer = concatUint8Array([buffer, value]); + } + } -export class Sparse { - /** - * @param {Blob} blob - * @param {Header} header - */ - constructor(blob, header) { - this.blob = blob; - this.header = header; + let header; + try { + await readUntil(FILE_HEADER_SIZE); + header = parseFileHeader(buffer.buffer); + if (header === null) return null; + buffer = buffer.slice(FILE_HEADER_SIZE); + } catch (e) { + reader.releaseLock(); + throw e; } - /** - * @returns {AsyncIterator} - */ - async* chunks() { - let blobOffset = FILE_HEADER_SIZE; - for (let i = 0; i < this.header.totalChunks; i++) { - if (blobOffset + CHUNK_HEADER_SIZE >= this.blob.size) { - throw "Sparse - Chunk header out of bounds"; + let chunkIndex = 0; + return [header, new ReadableStream({ + async pull(controller) { + await readUntil(CHUNK_HEADER_SIZE, controller); + while (buffer.byteLength >= CHUNK_HEADER_SIZE && chunkIndex < header.totalChunks) { + const view = new DataView(buffer.buffer); + const chunkType = view.getUint16(0, true); + const chunkBlockCount = view.getUint32(4, true); + const chunkTotalBytes = view.getUint32(8, true); + await readUntil(chunkTotalBytes, controller); + const chunkData = buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes); + controller.enqueue({ + type: chunkType, + blocks: chunkBlockCount, + data: chunkData, + }); + chunkIndex++; + buffer = buffer.slice(chunkTotalBytes); } - const chunk = await this.blob.slice(blobOffset, blobOffset + CHUNK_HEADER_SIZE).arrayBuffer(); - const view = new DataView(chunk); - const totalBytes = view.getUint32(8, true); - if (blobOffset + totalBytes > this.blob.size) { - throw "Sparse - Chunk data out of bounds"; + if (chunkIndex === header.totalChunks) { + controller.close(); + if (buffer.byteLength > 0) { + console.warn("Sparse - Backing data larger than expected"); + } } - yield { - type: view.getUint16(0, true), - blocks: view.getUint32(4, true), - data: this.blob.slice(blobOffset + CHUNK_HEADER_SIZE, blobOffset + totalBytes), - }; - blobOffset += totalBytes; - } - if (blobOffset !== this.blob.size) { - console.warn("Sparse - Backing data larger expected"); - } - } + }, + cancel() { + reader.releaseLock(); + }, + })]; +} - /** - * @returns {AsyncIterator<[number, Blob | null, number]>} - */ - async *read() { - let offset = 0; - for await (const { type, blocks, data } of this.chunks()) { - const size = blocks * this.header.blockSize; + +/** + * @param {SparseHeader} header + * @param {ReadableStream} stream + * @returns {ReadableStream<[number, Uint8Array | null, number]>} + */ +export function read(header, stream) { + const reader = stream.getReader(); + let offset = 0; + return new ReadableStream({ + async pull(controller) { + const { value, done } = await reader.read(); + if (done) { + controller.close(); + return; + } + const { type, blocks, data } = value; + const size = blocks * header.blockSize; if (type === ChunkType.Raw) { - yield [offset, data, size]; + controller.enqueue([offset, data, size]); offset += size; } else if (type === ChunkType.Fill) { - const fill = new Uint8Array(await data.arrayBuffer()); - if (fill.some((byte) => byte !== 0)) { + if (data.some((byte) => byte !== 0)) { const buffer = new Uint8Array(size); - for (let i = 0; i < buffer.byteLength; i += 4) buffer.set(fill, i); - yield [offset, new Blob([buffer]), size]; + for (let i = 0; i < buffer.byteLength; i += 4) buffer.set(data, i); + controller.enqueue([offset, buffer, size]); } else { - yield [offset, null, size]; + controller.enqueue([offset, null, size]); } offset += size; } else if (type === ChunkType.Skip) { - yield [offset, null, size]; + controller.enqueue([offset, null, size]); offset += size; } - } - } -} - - -/** - * @param {Blob} blob - * @returns {Promise} - */ -export async function from(blob) { - const header = parseFileHeader(await blob.slice(0, FILE_HEADER_SIZE).arrayBuffer()); - if (!header) return null; - return new Sparse(blob, header); + }, + cancel() { + reader.releaseLock(); + }, + }); } /** - * @param {Uint8Array} buffer - * @returns {Header|null} + * @param {ArrayBufferLike} buffer + * @returns {SparseHeader | null} */ export function parseFileHeader(buffer) { const view = new DataView(buffer); @@ -123,10 +149,10 @@ export function parseFileHeader(buffer) { const fileHeaderSize = view.getUint16(8, true); const chunkHeaderSize = view.getUint16(10, true); if (fileHeaderSize !== FILE_HEADER_SIZE) { - throw `Sparse - The file header size was expected to be 28, but is ${fileHeaderSize}.`; + throw new Error(`The file header size was expected to be 28, but is ${fileHeaderSize}.`); } if (chunkHeaderSize !== CHUNK_HEADER_SIZE) { - throw `Sparse - The chunk header size was expected to be 12, but is ${chunkHeaderSize}.`; + throw new Error(`The chunk header size was expected to be 12, but is ${chunkHeaderSize}.`); } return { magic, diff --git a/src/sparse.spec.js b/src/sparse.spec.js index 8e7d404..c9a46fb 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -1,5 +1,5 @@ import * as Bun from "bun"; -import { beforeAll, describe, expect, test } from "bun:test"; +import { describe, expect, test } from "bun:test"; import * as Sparse from "./sparse"; import { simg2img } from "../scripts/simg2img.js"; @@ -9,7 +9,7 @@ const expectedPath = "./test/fixtures/raw.img"; describe("sparse", () => { test("parseFileHeader", async () => { - expect(await Sparse.parseFileHeader(inputData)).toEqual({ + expect(await Sparse.parseFileHeader(await inputData.arrayBuffer())).toEqual({ magic: 0xED26FF3A, majorVersion: 1, minorVersion: 0, @@ -22,28 +22,10 @@ describe("sparse", () => { }); }); - describe("Sparse", () => { - /** @type {Sparse.Sparse} */ - let sparse; - - beforeAll(async () => { - sparse = await Sparse.from(inputData); - }); - - test("chunks", async () => { - const chunks = await Array.fromAsync(sparse.chunks()); - expect(chunks.length).toBe(sparse.header.totalChunks); - }); - - test("read", async () => { - let prevOffset = undefined; - for await (const [offset, chunk, size] of sparse.read()) { - expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); - if (chunk) expect(chunk.size).toBe(size); - expect(size).toBeGreaterThan(0); - prevOffset = offset + size; - } - }); + test("from", async () => { + const [header, chunks] = await Sparse.from(inputData.stream()); + const result = await Bun.readableStreamToArray(chunks); + expect(result.length).toBe(header.totalChunks); }); test("simg2img", async () => { From 7eaed6a4234496c8556d8ec3aff4b1b5867aa565 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 00:01:46 +0000 Subject: [PATCH 04/22] test --- src/sparse.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index b74bbdb..9cd3bb0 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -43,11 +43,16 @@ export async function from(stream) { let buffer = new Uint8Array(0); const readUntil = async (byteLength) => { - while (buffer.byteLength < byteLength) { + if (buffer.byteLength >= byteLength) return; + const parts = [buffer]; + let size = buffer.byteLength; + while (size < byteLength) { const { value, done } = await reader.read(); if (done) throw new Error("Unexpected end of stream"); - buffer = concatUint8Array([buffer, value]); + parts.push(value); + size += value.byteLength; } + buffer = concatUint8Array(parts); } let header; From 024dfde9a2df0aa2621a4c40336fb8ce6e70f0c4 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 00:04:01 +0000 Subject: [PATCH 05/22] cleanup --- src/sparse.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index 9cd3bb0..10cbfba 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -69,18 +69,17 @@ export async function from(stream) { let chunkIndex = 0; return [header, new ReadableStream({ async pull(controller) { - await readUntil(CHUNK_HEADER_SIZE, controller); + await readUntil(CHUNK_HEADER_SIZE); while (buffer.byteLength >= CHUNK_HEADER_SIZE && chunkIndex < header.totalChunks) { const view = new DataView(buffer.buffer); const chunkType = view.getUint16(0, true); const chunkBlockCount = view.getUint32(4, true); const chunkTotalBytes = view.getUint32(8, true); - await readUntil(chunkTotalBytes, controller); - const chunkData = buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes); + await readUntil(chunkTotalBytes); controller.enqueue({ type: chunkType, blocks: chunkBlockCount, - data: chunkData, + data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), }); chunkIndex++; buffer = buffer.slice(chunkTotalBytes); From b7ffcf7096d0626d2c1f3e799019ae10eba5062e Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 00:06:28 +0000 Subject: [PATCH 06/22] add back read test --- src/sparse.spec.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/sparse.spec.js b/src/sparse.spec.js index c9a46fb..317cb9f 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -28,6 +28,18 @@ describe("sparse", () => { expect(result.length).toBe(header.totalChunks); }); + test("read", async () => { + const [header, chunks] = await Sparse.from(inputData.stream()); + const stream = Sparse.read(header, chunks); + let prevOffset = undefined; + for (const [offset, chunk, size] of await Bun.readableStreamToArray(stream)) { + expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); + if (chunk) expect(chunk.byteLength).toBe(size); + expect(size).toBeGreaterThan(0); + prevOffset = offset + size; + } + }); + test("simg2img", async () => { const outputPath = `/tmp/${Bun.randomUUIDv7()}.img`; await simg2img(inputData.name, outputPath); From 26de18ccbde07bcc7aa6640d401c8426d0eb29fa Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 00:30:33 +0000 Subject: [PATCH 07/22] don't allow performance regressions --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45f6d30..226bbd7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: run: simg2img system.img system-raw.img - name: run sparse benchmark - run: bun scripts/simg2img.js system.img /tmp/system-raw.img + run: timeout 15s bun scripts/simg2img.js system.img /tmp/system-raw.img - name: check output matches run: cmp system-raw.img /tmp/system-raw.img From a03962f323ceb8bf2741698685713569bc74979a Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 00:57:06 +0000 Subject: [PATCH 08/22] refactor --- scripts/simg2img.js | 2 +- src/sparse.js | 13 +++++++------ src/sparse.spec.js | 8 ++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/simg2img.js b/scripts/simg2img.js index a405409..4a49ab8 100755 --- a/scripts/simg2img.js +++ b/scripts/simg2img.js @@ -11,7 +11,7 @@ export async function simg2img(inputPath, outputPath) { // FIXME: write out a "sparse" file? not supported by Bun const writer = outputImage.writer({ highWaterMark: 4 * 1024 * 1024 }); - const stream = Sparse.read(...result); + const stream = Sparse.read(result); const reader = stream.getReader(); while (true) { diff --git a/src/sparse.js b/src/sparse.js index 10cbfba..0cca822 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -28,6 +28,7 @@ const ChunkType = { /** * @typedef {object} SparseChunk + * @property {SparseHeader} header * @property {number} type * @property {number} blocks * @property {Uint8Array} data @@ -36,7 +37,7 @@ const ChunkType = { /** * @param {ReadableStream} stream - * @returns {Promise<[SparseHeader, ReadableStream] | null>} + * @returns {Promise | null>} */ export async function from(stream) { const reader = stream.getReader(); @@ -67,7 +68,7 @@ export async function from(stream) { } let chunkIndex = 0; - return [header, new ReadableStream({ + return new ReadableStream({ async pull(controller) { await readUntil(CHUNK_HEADER_SIZE); while (buffer.byteLength >= CHUNK_HEADER_SIZE && chunkIndex < header.totalChunks) { @@ -77,6 +78,7 @@ export async function from(stream) { const chunkTotalBytes = view.getUint32(8, true); await readUntil(chunkTotalBytes); controller.enqueue({ + header, type: chunkType, blocks: chunkBlockCount, data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), @@ -94,16 +96,15 @@ export async function from(stream) { cancel() { reader.releaseLock(); }, - })]; + }); } /** - * @param {SparseHeader} header * @param {ReadableStream} stream * @returns {ReadableStream<[number, Uint8Array | null, number]>} */ -export function read(header, stream) { +export function read(stream) { const reader = stream.getReader(); let offset = 0; return new ReadableStream({ @@ -113,7 +114,7 @@ export function read(header, stream) { controller.close(); return; } - const { type, blocks, data } = value; + const { header, type, blocks, data } = value; const size = blocks * header.blockSize; if (type === ChunkType.Raw) { controller.enqueue([offset, data, size]); diff --git a/src/sparse.spec.js b/src/sparse.spec.js index 317cb9f..d110050 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -23,14 +23,14 @@ describe("sparse", () => { }); test("from", async () => { - const [header, chunks] = await Sparse.from(inputData.stream()); + const chunks = await Sparse.from(inputData.stream()); const result = await Bun.readableStreamToArray(chunks); - expect(result.length).toBe(header.totalChunks); + expect(result.length).toBe(result[0].header.totalChunks); }); test("read", async () => { - const [header, chunks] = await Sparse.from(inputData.stream()); - const stream = Sparse.read(header, chunks); + const chunks = await Sparse.from(inputData.stream()); + const stream = Sparse.read(chunks); let prevOffset = undefined; for (const [offset, chunk, size] of await Bun.readableStreamToArray(stream)) { expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); From 0ae33bff6778832d627ee5d9fbd44278cd881b70 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:12:15 +0000 Subject: [PATCH 09/22] oh --- scripts/simg2img.js | 16 ++--- src/sparse.js | 141 ++++++++++++++++++-------------------------- src/sparse.spec.js | 12 ++-- 3 files changed, 67 insertions(+), 102 deletions(-) diff --git a/scripts/simg2img.js b/scripts/simg2img.js index 4a49ab8..4cf7354 100755 --- a/scripts/simg2img.js +++ b/scripts/simg2img.js @@ -5,28 +5,20 @@ export async function simg2img(inputPath, outputPath) { const sparseImage = Bun.file(inputPath); const outputImage = Bun.file(outputPath); - const result = await Sparse.from(sparseImage.stream()); - if (!result) throw "Failed to parse sparse file"; + const chunks = await Sparse.readChunks(sparseImage.stream()); + if (!chunks) throw "Failed to parse sparse file"; // FIXME: write out a "sparse" file? not supported by Bun const writer = outputImage.writer({ highWaterMark: 4 * 1024 * 1024 }); - - const stream = Sparse.read(result); - const reader = stream.getReader(); - - while (true) { - const { value, done } = await reader.read(); - if (done) break; - const [_, data, size ] = value; + for await (const [_, data, size] of Sparse.inflateChunks(chunks)) { if (data) { - writer.write(data); + writer.write(data.buffer); } else { writer.write(new Uint8Array(size).buffer); } } writer.end(); - reader.releaseLock(); } if (import.meta.main) { diff --git a/src/sparse.js b/src/sparse.js index 0cca822..2ebc20c 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -37,106 +37,81 @@ const ChunkType = { /** * @param {ReadableStream} stream - * @returns {Promise | null>} + * @returns {Promise | null>} */ -export async function from(stream) { - const reader = stream.getReader(); +export async function* readChunks(stream) { let buffer = new Uint8Array(0); const readUntil = async (byteLength) => { if (buffer.byteLength >= byteLength) return; - const parts = [buffer]; - let size = buffer.byteLength; - while (size < byteLength) { - const { value, done } = await reader.read(); - if (done) throw new Error("Unexpected end of stream"); - parts.push(value); - size += value.byteLength; + const reader = stream.getReader(); + try { + const parts = [buffer]; + let size = buffer.byteLength; + while (size < byteLength) { + const { value, done } = await reader.read(); + if (done) throw new Error("Unexpected end of stream"); + parts.push(value); + size += value.byteLength; + } + buffer = concatUint8Array(parts); + } finally { + reader.releaseLock(); } - buffer = concatUint8Array(parts); } - let header; - try { - await readUntil(FILE_HEADER_SIZE); - header = parseFileHeader(buffer.buffer); - if (header === null) return null; - buffer = buffer.slice(FILE_HEADER_SIZE); - } catch (e) { - reader.releaseLock(); - throw e; + await readUntil(FILE_HEADER_SIZE); + const header = parseFileHeader(buffer.buffer); + if (header === null) return null; + buffer = buffer.slice(FILE_HEADER_SIZE); + + for (let i = 0; i < header.totalChunks; i++) { + await readUntil(CHUNK_HEADER_SIZE); + const view = new DataView(buffer.buffer); + const chunkType = view.getUint16(0, true); + const chunkBlockCount = view.getUint32(4, true); + const chunkTotalBytes = view.getUint32(8, true); + await readUntil(chunkTotalBytes); + yield { + header, + type: chunkType, + blocks: chunkBlockCount, + data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), + }; + buffer = buffer.slice(chunkTotalBytes); } - let chunkIndex = 0; - return new ReadableStream({ - async pull(controller) { - await readUntil(CHUNK_HEADER_SIZE); - while (buffer.byteLength >= CHUNK_HEADER_SIZE && chunkIndex < header.totalChunks) { - const view = new DataView(buffer.buffer); - const chunkType = view.getUint16(0, true); - const chunkBlockCount = view.getUint32(4, true); - const chunkTotalBytes = view.getUint32(8, true); - await readUntil(chunkTotalBytes); - controller.enqueue({ - header, - type: chunkType, - blocks: chunkBlockCount, - data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), - }); - chunkIndex++; - buffer = buffer.slice(chunkTotalBytes); - } - if (chunkIndex === header.totalChunks) { - controller.close(); - if (buffer.byteLength > 0) { - console.warn("Sparse - Backing data larger than expected"); - } - } - }, - cancel() { - reader.releaseLock(); - }, - }); + if (buffer.byteLength > 0) { + console.warn("Sparse - Backing data larger than expected"); + } } /** - * @param {ReadableStream} stream - * @returns {ReadableStream<[number, Uint8Array | null, number]>} + * @param {AsyncIterator} chunks + * @returns {AsyncIterator<[number, Uint8Array | null, number]>} */ -export function read(stream) { - const reader = stream.getReader(); +export async function* inflateChunks(chunks) { let offset = 0; - return new ReadableStream({ - async pull(controller) { - const { value, done } = await reader.read(); - if (done) { - controller.close(); - return; - } - const { header, type, blocks, data } = value; - const size = blocks * header.blockSize; - if (type === ChunkType.Raw) { - controller.enqueue([offset, data, size]); - offset += size; - } else if (type === ChunkType.Fill) { - if (data.some((byte) => byte !== 0)) { - const buffer = new Uint8Array(size); - for (let i = 0; i < buffer.byteLength; i += 4) buffer.set(data, i); - controller.enqueue([offset, buffer, size]); - } else { - controller.enqueue([offset, null, size]); - } - offset += size; - } else if (type === ChunkType.Skip) { - controller.enqueue([offset, null, size]); - offset += size; + for await (const { header, type, blocks, data } of chunks) { + const size = blocks * header.blockSize; + if (type === ChunkType.Raw) { + yield [offset, data, size]; + offset += size; + } else if (type === ChunkType.Fill) { + if (data.some((byte) => byte !== 0)) { + const buffer = new Uint8Array(size); + for (let i = 0; i < buffer.byteLength; i += 4) buffer.set(data, i); + yield [offset, buffer, size]; + } else { + yield [offset, null, size]; } - }, - cancel() { - reader.releaseLock(); - }, - }); + offset += size; + } else if (type === ChunkType.Skip) { + yield [offset, null, size]; + offset += size; + } + } } diff --git a/src/sparse.spec.js b/src/sparse.spec.js index d110050..f08f727 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -23,18 +23,16 @@ describe("sparse", () => { }); test("from", async () => { - const chunks = await Sparse.from(inputData.stream()); - const result = await Bun.readableStreamToArray(chunks); - expect(result.length).toBe(result[0].header.totalChunks); + const chunks = await Array.fromAsync(await Sparse.readChunks(inputData.stream())); + expect(chunks.length).toBe(chunks[0].header.totalChunks); }); test("read", async () => { - const chunks = await Sparse.from(inputData.stream()); - const stream = Sparse.read(chunks); + const chunks = await Sparse.readChunks(inputData.stream()); let prevOffset = undefined; - for (const [offset, chunk, size] of await Bun.readableStreamToArray(stream)) { + for await (const [offset, data, size] of Sparse.inflateChunks(chunks)) { expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); - if (chunk) expect(chunk.byteLength).toBe(size); + if (data) expect(data.byteLength).toBe(size); expect(size).toBeGreaterThan(0); prevOffset = offset + size; } From 4becb6bbf615f04e1556501f185981696a4f513e Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:16:31 +0000 Subject: [PATCH 10/22] rename --- src/sparse.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sparse.spec.js b/src/sparse.spec.js index f08f727..2869f9a 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -22,12 +22,12 @@ describe("sparse", () => { }); }); - test("from", async () => { + test("readChunks", async () => { const chunks = await Array.fromAsync(await Sparse.readChunks(inputData.stream())); expect(chunks.length).toBe(chunks[0].header.totalChunks); }); - test("read", async () => { + test("inflateChunks", async () => { const chunks = await Sparse.readChunks(inputData.stream()); let prevOffset = undefined; for await (const [offset, data, size] of Sparse.inflateChunks(chunks)) { From 5c370fee1309f6d50b48b44f74acfac9ae6e6a3a Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:16:53 +0000 Subject: [PATCH 11/22] . --- scripts/simg2img.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/simg2img.js b/scripts/simg2img.js index 4cf7354..1794721 100755 --- a/scripts/simg2img.js +++ b/scripts/simg2img.js @@ -17,7 +17,6 @@ export async function simg2img(inputPath, outputPath) { writer.write(new Uint8Array(size).buffer); } } - writer.end(); } From 724675a0fd0df74d7861e7b3d5d1d127d7309b39 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:18:08 +0000 Subject: [PATCH 12/22] rm --- src/sparse.js | 6 +++--- src/sparse.spec.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index 2ebc20c..d1734c9 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -13,7 +13,7 @@ const ChunkType = { /** - * @typedef {object} SparseHeader + * @typedef {object} Header * @property {number} magic * @property {number} majorVersion * @property {number} minorVersion @@ -28,7 +28,7 @@ const ChunkType = { /** * @typedef {object} SparseChunk - * @property {SparseHeader} header + * @property {Header} header * @property {number} type * @property {number} blocks * @property {Uint8Array} data @@ -117,7 +117,7 @@ export async function* inflateChunks(chunks) { /** * @param {ArrayBufferLike} buffer - * @returns {SparseHeader | null} + * @returns {Header | null} */ export function parseFileHeader(buffer) { const view = new DataView(buffer); diff --git a/src/sparse.spec.js b/src/sparse.spec.js index 2869f9a..1ccb83d 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -9,7 +9,7 @@ const expectedPath = "./test/fixtures/raw.img"; describe("sparse", () => { test("parseFileHeader", async () => { - expect(await Sparse.parseFileHeader(await inputData.arrayBuffer())).toEqual({ + expect(Sparse.parseFileHeader(await inputData.arrayBuffer())).toEqual({ magic: 0xED26FF3A, majorVersion: 1, minorVersion: 0, From c09f09b24ac455fdff782e06a6089d93c49d562d Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:19:45 +0000 Subject: [PATCH 13/22] use new API in firehose --- src/qdl.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/qdl.js b/src/qdl.js index c942af3..70005a7 100644 --- a/src/qdl.js +++ b/src/qdl.js @@ -116,8 +116,8 @@ export class qdlDevice { } console.info(`Flashing ${partitionName}...`); console.debug(`startSector ${partition.sector}, sectors ${partition.sectors}`); - const sparse = await Sparse.from(blob); - if (sparse === null) { + const chunks = await Sparse.readChunks(blob.stream()); + if (chunks === null) { return await this.firehose.cmdProgram(lun, partition.sector, blob, onProgress); } console.debug(`Erasing ${partitionName}...`); @@ -126,7 +126,7 @@ export class qdlDevice { return false; } console.debug(`Writing chunks to ${partitionName}...`); - for await (const [offset, chunk] of sparse.read()) { + for await (const [offset, chunk] of Sparse.inflateChunks(chunks)) { if (!chunk) continue; if (offset % this.firehose.cfg.SECTOR_SIZE_IN_BYTES !== 0) { throw "qdl - Offset not aligned to sector size"; From d5cb3723421b56cfa943b96cc8b4880bb372f5b0 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:37:33 +0000 Subject: [PATCH 14/22] refactor --- scripts/simg2img.js | 2 +- src/qdl.js | 2 +- src/sparse.js | 49 +++++++++++++++++++++++++++------------------ src/sparse.spec.js | 34 ++++++++++++++++++------------- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/scripts/simg2img.js b/scripts/simg2img.js index 1794721..782dce0 100755 --- a/scripts/simg2img.js +++ b/scripts/simg2img.js @@ -5,7 +5,7 @@ export async function simg2img(inputPath, outputPath) { const sparseImage = Bun.file(inputPath); const outputImage = Bun.file(outputPath); - const chunks = await Sparse.readChunks(sparseImage.stream()); + const chunks = await Sparse.from(sparseImage.stream()); if (!chunks) throw "Failed to parse sparse file"; // FIXME: write out a "sparse" file? not supported by Bun diff --git a/src/qdl.js b/src/qdl.js index 70005a7..0d6e058 100644 --- a/src/qdl.js +++ b/src/qdl.js @@ -116,7 +116,7 @@ export class qdlDevice { } console.info(`Flashing ${partitionName}...`); console.debug(`startSector ${partition.sector}, sectors ${partition.sectors}`); - const chunks = await Sparse.readChunks(blob.stream()); + const chunks = Sparse.readChunks(blob.stream()); if (chunks === null) { return await this.firehose.cmdProgram(lun, partition.sector, blob, onProgress); } diff --git a/src/sparse.js b/src/sparse.js index d1734c9..a4b7895 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -37,11 +37,14 @@ const ChunkType = { /** * @param {ReadableStream} stream - * @returns {Promise | null>} + * @returns {AsyncGenerator | null} */ -export async function* readChunks(stream) { +export async function from(stream) { let buffer = new Uint8Array(0); + /** + * @param {number} byteLength + */ const readUntil = async (byteLength) => { if (buffer.byteLength >= byteLength) return; const reader = stream.getReader(); @@ -62,28 +65,34 @@ export async function* readChunks(stream) { await readUntil(FILE_HEADER_SIZE); const header = parseFileHeader(buffer.buffer); - if (header === null) return null; + if (!header) return null; buffer = buffer.slice(FILE_HEADER_SIZE); - for (let i = 0; i < header.totalChunks; i++) { - await readUntil(CHUNK_HEADER_SIZE); - const view = new DataView(buffer.buffer); - const chunkType = view.getUint16(0, true); - const chunkBlockCount = view.getUint32(4, true); - const chunkTotalBytes = view.getUint32(8, true); - await readUntil(chunkTotalBytes); - yield { - header, - type: chunkType, - blocks: chunkBlockCount, - data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), - }; - buffer = buffer.slice(chunkTotalBytes); + /** + * @returns {AsyncGenerator} + */ + async function* readChunks() { + for (let i = 0; i < header.totalChunks; i++) { + await readUntil(CHUNK_HEADER_SIZE); + const view = new DataView(buffer.buffer); + const chunkType = view.getUint16(0, true); + const chunkBlockCount = view.getUint32(4, true); + const chunkTotalBytes = view.getUint32(8, true); + await readUntil(chunkTotalBytes); + yield { + header, + type: chunkType, + blocks: chunkBlockCount, + data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), + }; + buffer = buffer.slice(chunkTotalBytes); + } + if (buffer.byteLength > 0) { + console.warn("Sparse - Backing data larger than expected"); + } } - if (buffer.byteLength > 0) { - console.warn("Sparse - Backing data larger than expected"); - } + return readChunks(); } diff --git a/src/sparse.spec.js b/src/sparse.spec.js index 1ccb83d..0475399 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -8,27 +8,33 @@ const inputData = Bun.file("./test/fixtures/sparse.img"); const expectedPath = "./test/fixtures/raw.img"; describe("sparse", () => { - test("parseFileHeader", async () => { - expect(Sparse.parseFileHeader(await inputData.arrayBuffer())).toEqual({ - magic: 0xED26FF3A, - majorVersion: 1, - minorVersion: 0, - fileHeaderSize: 28, - chunkHeaderSize: 12, - blockSize: 4096, - totalBlocks: 9, - totalChunks: 6, - crc32: 0, + describe("parseFileHeader", () => { + test("valid sparse file", async () => { + expect(Sparse.parseFileHeader(await inputData.arrayBuffer())).toEqual({ + magic: 0xED26FF3A, + majorVersion: 1, + minorVersion: 0, + fileHeaderSize: 28, + chunkHeaderSize: 12, + blockSize: 4096, + totalBlocks: 9, + totalChunks: 6, + crc32: 0, + }); + }); + + test("invalid sparse file", async () => { + expect(Sparse.parseFileHeader(await Bun.file(expectedPath).arrayBuffer())).toBeNull(); }); }); - test("readChunks", async () => { - const chunks = await Array.fromAsync(await Sparse.readChunks(inputData.stream())); + test("from", async () => { + const chunks = await Array.fromAsync(await Sparse.from(inputData.stream())); expect(chunks.length).toBe(chunks[0].header.totalChunks); }); test("inflateChunks", async () => { - const chunks = await Sparse.readChunks(inputData.stream()); + const chunks = await Sparse.from(inputData.stream()); let prevOffset = undefined; for await (const [offset, data, size] of Sparse.inflateChunks(chunks)) { expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); From 118bfb694b5e3536a961cda65fea506b137a8cf8 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:38:15 +0000 Subject: [PATCH 15/22] fix return type --- src/qdl.js | 2 +- src/sparse.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/qdl.js b/src/qdl.js index 0d6e058..cbd34a0 100644 --- a/src/qdl.js +++ b/src/qdl.js @@ -116,7 +116,7 @@ export class qdlDevice { } console.info(`Flashing ${partitionName}...`); console.debug(`startSector ${partition.sector}, sectors ${partition.sectors}`); - const chunks = Sparse.readChunks(blob.stream()); + const chunks = await Sparse.from(blob.stream()); if (chunks === null) { return await this.firehose.cmdProgram(lun, partition.sector, blob, onProgress); } diff --git a/src/sparse.js b/src/sparse.js index a4b7895..2083763 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -37,7 +37,7 @@ const ChunkType = { /** * @param {ReadableStream} stream - * @returns {AsyncGenerator | null} + * @returns {Promise | null>} */ export async function from(stream) { let buffer = new Uint8Array(0); From 85c052dc2bda4cafdb69913da8db335a4e40c879 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:38:59 +0000 Subject: [PATCH 16/22] fix --- src/sparse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sparse.js b/src/sparse.js index 2083763..ed28fc6 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -37,7 +37,7 @@ const ChunkType = { /** * @param {ReadableStream} stream - * @returns {Promise | null>} + * @returns {Promise | null>} */ export async function from(stream) { let buffer = new Uint8Array(0); From 80e30e48fa48284ab7e89406ec51ffacec27bb5e Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 01:42:29 +0000 Subject: [PATCH 17/22] to blob --- src/qdl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qdl.js b/src/qdl.js index cbd34a0..2d38376 100644 --- a/src/qdl.js +++ b/src/qdl.js @@ -133,7 +133,7 @@ export class qdlDevice { } const sector = partition.sector + offset / this.firehose.cfg.SECTOR_SIZE_IN_BYTES; const onChunkProgress = (progress) => onProgress?.(offset + progress); - if (!await this.firehose.cmdProgram(lun, sector, chunk, onChunkProgress)) { + if (!await this.firehose.cmdProgram(lun, sector, new Blob([chunk]), onChunkProgress)) { console.debug("qdl - Failed to program chunk") return false; } From 9d6fea9bfed0a7f5c543fe4a3908cafdf960f5d8 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 20:48:12 +0000 Subject: [PATCH 18/22] all at once --- scripts/simg2img.js | 6 +-- src/qdl.js | 10 ++--- src/sparse.js | 102 +++++++++++++++++++++++++------------------- src/sparse.spec.js | 9 +--- 4 files changed, 68 insertions(+), 59 deletions(-) diff --git a/scripts/simg2img.js b/scripts/simg2img.js index 782dce0..f0a9922 100755 --- a/scripts/simg2img.js +++ b/scripts/simg2img.js @@ -5,12 +5,12 @@ export async function simg2img(inputPath, outputPath) { const sparseImage = Bun.file(inputPath); const outputImage = Bun.file(outputPath); - const chunks = await Sparse.from(sparseImage.stream()); - if (!chunks) throw "Failed to parse sparse file"; + const sparse = await Sparse.from(sparseImage.stream()); + if (!sparse) throw "Failed to parse sparse file"; // FIXME: write out a "sparse" file? not supported by Bun const writer = outputImage.writer({ highWaterMark: 4 * 1024 * 1024 }); - for await (const [_, data, size] of Sparse.inflateChunks(chunks)) { + for await (const [_, data, size] of sparse) { if (data) { writer.write(data.buffer); } else { diff --git a/src/qdl.js b/src/qdl.js index 2d38376..57bf5c1 100644 --- a/src/qdl.js +++ b/src/qdl.js @@ -116,8 +116,8 @@ export class qdlDevice { } console.info(`Flashing ${partitionName}...`); console.debug(`startSector ${partition.sector}, sectors ${partition.sectors}`); - const chunks = await Sparse.from(blob.stream()); - if (chunks === null) { + const sparse = await Sparse.from(blob.stream()); + if (sparse === null) { return await this.firehose.cmdProgram(lun, partition.sector, blob, onProgress); } console.debug(`Erasing ${partitionName}...`); @@ -126,14 +126,14 @@ export class qdlDevice { return false; } console.debug(`Writing chunks to ${partitionName}...`); - for await (const [offset, chunk] of Sparse.inflateChunks(chunks)) { - if (!chunk) continue; + for await (const [offset, data] of sparse) { + if (!data) continue; if (offset % this.firehose.cfg.SECTOR_SIZE_IN_BYTES !== 0) { throw "qdl - Offset not aligned to sector size"; } const sector = partition.sector + offset / this.firehose.cfg.SECTOR_SIZE_IN_BYTES; const onChunkProgress = (progress) => onProgress?.(offset + progress); - if (!await this.firehose.cmdProgram(lun, sector, new Blob([chunk]), onChunkProgress)) { + if (!await this.firehose.cmdProgram(lun, sector, new Blob([data]), onChunkProgress)) { console.debug("qdl - Failed to program chunk") return false; } diff --git a/src/sparse.js b/src/sparse.js index ed28fc6..bccd47f 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -35,11 +35,18 @@ const ChunkType = { */ +function assert(condition) { + if (!condition) throw new Error("Assertion failed"); +} + + /** * @param {ReadableStream} stream - * @returns {Promise | null>} + * @param {number} maxSize + * @param {number} sectorSize + * @returns {Promise | null>} */ -export async function from(stream) { +export async function from(stream, maxSize = 1024 * 1024, sectorSize = 1024) { let buffer = new Uint8Array(0); /** @@ -69,58 +76,65 @@ export async function from(stream) { buffer = buffer.slice(FILE_HEADER_SIZE); /** - * @returns {AsyncGenerator} + * @returns {AsyncGenerator<[number, Uint8Array | null, number], void, *>} */ - async function* readChunks() { + async function* inflateChunks() { + let offset = 0; for (let i = 0; i < header.totalChunks; i++) { await readUntil(CHUNK_HEADER_SIZE); const view = new DataView(buffer.buffer); - const chunkType = view.getUint16(0, true); - const chunkBlockCount = view.getUint32(4, true); - const chunkTotalBytes = view.getUint32(8, true); - await readUntil(chunkTotalBytes); - yield { - header, - type: chunkType, - blocks: chunkBlockCount, - data: buffer.slice(CHUNK_HEADER_SIZE, chunkTotalBytes), - }; - buffer = buffer.slice(chunkTotalBytes); + const type = view.getUint16(0, true); + const blockCount = view.getUint32(4, true); + const totalBytes = view.getUint32(8, true); + + const size = blockCount * header.blockSize; + + if (type === ChunkType.Raw) { + assert(size === totalBytes - CHUNK_HEADER_SIZE); + + let readBytes = 0; + buffer = buffer.slice(CHUNK_HEADER_SIZE); + while (readBytes < totalBytes - CHUNK_HEADER_SIZE) { + let dataChunkSize = Math.min(totalBytes - CHUNK_HEADER_SIZE - readBytes, maxSize); + await readUntil(dataChunkSize); + if (totalBytes - CHUNK_HEADER_SIZE - readBytes > maxSize) { + dataChunkSize = sectorSize * Math.floor(dataChunkSize / sectorSize) + } + yield [offset, buffer.slice(0, dataChunkSize), size]; + buffer = buffer.slice(dataChunkSize); + readBytes += dataChunkSize; + } + assert(readBytes === size); + offset += size; + } else if (type === ChunkType.Fill) { + await readUntil(totalBytes); + const data = buffer.slice(CHUNK_HEADER_SIZE, totalBytes); + buffer = buffer.slice(totalBytes); + if (data.some((byte) => byte !== 0)) { + // FIXME: yield maxSize chunks + const fill = new Uint8Array(size); + for (let i = 0; i < data.byteLength; i += 4) fill.set(data, i); + yield [offset, fill, size]; + } else { + yield [offset, null, size]; + } + offset += size; + } else { + assert(type === ChunkType.Skip); + if (type === ChunkType.Skip) { + yield [offset, null, size]; + offset += size; + } + await readUntil(totalBytes); + buffer = buffer.slice(totalBytes); + } } if (buffer.byteLength > 0) { console.warn("Sparse - Backing data larger than expected"); } } - return readChunks(); -} - - -/** - * @param {AsyncIterator} chunks - * @returns {AsyncIterator<[number, Uint8Array | null, number]>} - */ -export async function* inflateChunks(chunks) { - let offset = 0; - for await (const { header, type, blocks, data } of chunks) { - const size = blocks * header.blockSize; - if (type === ChunkType.Raw) { - yield [offset, data, size]; - offset += size; - } else if (type === ChunkType.Fill) { - if (data.some((byte) => byte !== 0)) { - const buffer = new Uint8Array(size); - for (let i = 0; i < buffer.byteLength; i += 4) buffer.set(data, i); - yield [offset, buffer, size]; - } else { - yield [offset, null, size]; - } - offset += size; - } else if (type === ChunkType.Skip) { - yield [offset, null, size]; - offset += size; - } - } + return inflateChunks(); } diff --git a/src/sparse.spec.js b/src/sparse.spec.js index 0475399..13f19e9 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -29,14 +29,9 @@ describe("sparse", () => { }); test("from", async () => { - const chunks = await Array.fromAsync(await Sparse.from(inputData.stream())); - expect(chunks.length).toBe(chunks[0].header.totalChunks); - }); - - test("inflateChunks", async () => { - const chunks = await Sparse.from(inputData.stream()); + const sparse = await Sparse.from(inputData.stream()); let prevOffset = undefined; - for await (const [offset, data, size] of Sparse.inflateChunks(chunks)) { + for await (const [offset, data, size] of sparse) { expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); if (data) expect(data.byteLength).toBe(size); expect(size).toBeGreaterThan(0); From b8ee7ec45ac85006fa3d7eb0994dd4c9d2f14ad5 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 21:24:31 +0000 Subject: [PATCH 19/22] better test --- src/sparse.spec.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sparse.spec.js b/src/sparse.spec.js index 13f19e9..925a9e4 100644 --- a/src/sparse.spec.js +++ b/src/sparse.spec.js @@ -30,12 +30,13 @@ describe("sparse", () => { test("from", async () => { const sparse = await Sparse.from(inputData.stream()); - let prevOffset = undefined; + if (!sparse) throw "Failed to parse sparse"; + let expectedOffset = 0; for await (const [offset, data, size] of sparse) { - expect(offset).toBeGreaterThanOrEqual(prevOffset ?? 0); + expect(offset).toBe(expectedOffset); if (data) expect(data.byteLength).toBe(size); expect(size).toBeGreaterThan(0); - prevOffset = offset + size; + expectedOffset = offset + size; } }); From 70c888400e85c5b8ef73aa230cd9f4e6cdb53c1d Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 21:33:02 +0000 Subject: [PATCH 20/22] fix fill --- src/sparse.js | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index bccd47f..e439eda 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -43,10 +43,9 @@ function assert(condition) { /** * @param {ReadableStream} stream * @param {number} maxSize - * @param {number} sectorSize * @returns {Promise | null>} */ -export async function from(stream, maxSize = 1024 * 1024, sectorSize = 1024) { +export async function from(stream, maxSize = 1024 * 1024) { let buffer = new Uint8Array(0); /** @@ -86,41 +85,37 @@ export async function from(stream, maxSize = 1024 * 1024, sectorSize = 1024) { const type = view.getUint16(0, true); const blockCount = view.getUint32(4, true); const totalBytes = view.getUint32(8, true); - const size = blockCount * header.blockSize; if (type === ChunkType.Raw) { - assert(size === totalBytes - CHUNK_HEADER_SIZE); - - let readBytes = 0; buffer = buffer.slice(CHUNK_HEADER_SIZE); - while (readBytes < totalBytes - CHUNK_HEADER_SIZE) { - let dataChunkSize = Math.min(totalBytes - CHUNK_HEADER_SIZE - readBytes, maxSize); + let readBytes = 0; + while (readBytes < size) { + const dataChunkSize = Math.min(size - readBytes, maxSize); await readUntil(dataChunkSize); - if (totalBytes - CHUNK_HEADER_SIZE - readBytes > maxSize) { - dataChunkSize = sectorSize * Math.floor(dataChunkSize / sectorSize) - } - yield [offset, buffer.slice(0, dataChunkSize), size]; + const data = buffer.slice(0, dataChunkSize); + assert(data.byteLength === dataChunkSize); + yield [offset, data, dataChunkSize]; buffer = buffer.slice(dataChunkSize); readBytes += dataChunkSize; + offset += dataChunkSize; } assert(readBytes === size); - offset += size; } else if (type === ChunkType.Fill) { await readUntil(totalBytes); const data = buffer.slice(CHUNK_HEADER_SIZE, totalBytes); buffer = buffer.slice(totalBytes); if (data.some((byte) => byte !== 0)) { - // FIXME: yield maxSize chunks + assert(data.byteLength === 4); + // FIXME: yield in maxSize chunks const fill = new Uint8Array(size); - for (let i = 0; i < data.byteLength; i += 4) fill.set(data, i); + for (let i = 0; i < fill.byteLength; i += 4) fill.set(data, i); yield [offset, fill, size]; } else { yield [offset, null, size]; } offset += size; } else { - assert(type === ChunkType.Skip); if (type === ChunkType.Skip) { yield [offset, null, size]; offset += size; From 4b474a62d84a508fddce8ea3d0d3775349d391f9 Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Fri, 7 Mar 2025 22:45:54 +0000 Subject: [PATCH 21/22] it's so slow now --- src/firehose.js | 26 ++++++++++++++------------ src/qdl.js | 13 +++++-------- src/sparse.js | 16 +++++++++++----- src/usblib.js | 2 +- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/firehose.js b/src/firehose.js index fbfcda7..54d70ae 100644 --- a/src/firehose.js +++ b/src/firehose.js @@ -188,22 +188,23 @@ export class Firehose { /** * @param {number} physicalPartitionNumber * @param {number} startSector - * @param {Blob} blob + * @param {Uint8Array} data * @param {progressCallback|undefined} [onProgress] - Returns number of bytes written * @returns {Promise} */ - async cmdProgram(physicalPartitionNumber, startSector, blob, onProgress = undefined) { - const total = blob.size; - - const rsp = await this.xmlSend(toXml("program", { + async cmdProgram(physicalPartitionNumber, startSector, data, onProgress = undefined) { + const total = data.byteLength; + const attributes = { SECTOR_SIZE_IN_BYTES: this.cfg.SECTOR_SIZE_IN_BYTES, num_partition_sectors: Math.ceil(total / this.cfg.SECTOR_SIZE_IN_BYTES), physical_partition_number: physicalPartitionNumber, start_sector: startSector, - })); + }; + + const rsp = await this.xmlSend(toXml("program", attributes)); if (!rsp.resp) { - console.error("Firehose - Failed to program"); - return false; + console.error("Firehose - Failed to program", attributes, rsp); + throw new Error("Failed to program"); } let i = 0; @@ -212,11 +213,12 @@ export class Firehose { while (bytesToWrite > 0) { const wlen = Math.min(bytesToWrite, this.cfg.MaxPayloadSizeToTargetInBytes); - let wdata = new Uint8Array(await blob.slice(offset, offset + wlen).arrayBuffer()); + let wdata = data.subarray(offset, offset + wlen); if (wlen % this.cfg.SECTOR_SIZE_IN_BYTES !== 0) { - const fillLen = (Math.floor(wlen / this.cfg.SECTOR_SIZE_IN_BYTES) + 1) * this.cfg.SECTOR_SIZE_IN_BYTES; - const fillArray = new Uint8Array(fillLen - wlen).fill(0x00); - wdata = concatUint8Array([wdata, fillArray]); + const fillLen = Math.ceil(wlen / this.cfg.SECTOR_SIZE_IN_BYTES) * this.cfg.SECTOR_SIZE_IN_BYTES; + const fillArray = new Uint8Array(fillLen); + fillArray.set(wdata); + wdata = fillArray; } await this.cdc.write(wdata); await this.cdc.write(new Uint8Array(0)); diff --git a/src/qdl.js b/src/qdl.js index 57bf5c1..9afafdf 100644 --- a/src/qdl.js +++ b/src/qdl.js @@ -118,12 +118,11 @@ export class qdlDevice { console.debug(`startSector ${partition.sector}, sectors ${partition.sectors}`); const sparse = await Sparse.from(blob.stream()); if (sparse === null) { - return await this.firehose.cmdProgram(lun, partition.sector, blob, onProgress); + return this.firehose.cmdProgram(lun, partition.sector, new Uint8Array(await blob.arrayBuffer()), onProgress); } console.debug(`Erasing ${partitionName}...`); if (!await this.firehose.cmdErase(lun, partition.sector, partition.sectors)) { - console.error("qdl - Failed to erase partition before sparse flashing"); - return false; + throw new Error("Failed to erase partition before sparse flashing"); } console.debug(`Writing chunks to ${partitionName}...`); for await (const [offset, data] of sparse) { @@ -133,7 +132,7 @@ export class qdlDevice { } const sector = partition.sector + offset / this.firehose.cfg.SECTOR_SIZE_IN_BYTES; const onChunkProgress = (progress) => onProgress?.(offset + progress); - if (!await this.firehose.cmdProgram(lun, sector, new Blob([data]), onChunkProgress)) { + if (!await this.firehose.cmdProgram(lun, sector, data, onChunkProgress)) { console.debug("qdl - Failed to program chunk") return false; } @@ -319,11 +318,9 @@ export class qdlDevice { continue; } const writeOffset = this.firehose.cfg.SECTOR_SIZE_IN_BYTES; - const gptBlobA = new Blob([gptDataA.slice(writeOffset)]); - await this.firehose.cmdProgram(lunA, 1, gptBlobA); + await this.firehose.cmdProgram(lunA, 1, gptDataA.slice(writeOffset)); if (!sameLun) { - const gptBlobB = new Blob([gptDataB.slice(writeOffset)]); - await this.firehose.cmdProgram(lunB, 1, gptBlobB); + await this.firehose.cmdProgram(lunB, 1, gptDataB.slice(writeOffset)); } } const activeBootLunId = (slot === "a") ? 1 : 2; diff --git a/src/sparse.js b/src/sparse.js index e439eda..f0c801b 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -107,14 +107,20 @@ export async function from(stream, maxSize = 1024 * 1024) { buffer = buffer.slice(totalBytes); if (data.some((byte) => byte !== 0)) { assert(data.byteLength === 4); - // FIXME: yield in maxSize chunks - const fill = new Uint8Array(size); - for (let i = 0; i < fill.byteLength; i += 4) fill.set(data, i); - yield [offset, fill, size]; + let readBytes = 0; + while (readBytes < size) { + const fillSize = Math.min(size - readBytes, maxSize); + const fill = new Uint8Array(fillSize); + for (let i = 0; i < fillSize; i += 4) fill.set(data, i); + yield [offset, fill, fillSize]; + offset += fillSize; + readBytes += fillSize; + } + assert(readBytes === size); } else { yield [offset, null, size]; + offset += size; } - offset += size; } else { if (type === ChunkType.Skip) { yield [offset, null, size]; diff --git a/src/usblib.js b/src/usblib.js index 0adb821..84625dd 100644 --- a/src/usblib.js +++ b/src/usblib.js @@ -121,7 +121,7 @@ export class usbClass { async write(data, wait = true) { let offset = 0; do { - const chunk = data.slice(offset, offset + constants.BULK_TRANSFER_SIZE); + const chunk = data.subarray(offset, offset + constants.BULK_TRANSFER_SIZE); offset += chunk.byteLength; const promise = this.device?.transferOut(this.epOut?.endpointNumber, chunk); // this is a hack, webusb doesn't have timed out catching From bf2b9bd33d04050189d58882aec8e47a7d60db2c Mon Sep 17 00:00:00 2001 From: Cameron Clough Date: Sat, 8 Mar 2025 14:18:21 +0000 Subject: [PATCH 22/22] hmmm --- src/sparse.js | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/sparse.js b/src/sparse.js index f0c801b..ad03be0 100644 --- a/src/sparse.js +++ b/src/sparse.js @@ -1,5 +1,3 @@ -import { concatUint8Array } from "./utils"; - const FILE_MAGIC = 0xed26ff3a; export const FILE_HEADER_SIZE = 28; const CHUNK_HEADER_SIZE = 12; @@ -46,27 +44,33 @@ function assert(condition) { * @returns {Promise | null>} */ export async function from(stream, maxSize = 1024 * 1024) { - let buffer = new Uint8Array(0); + let buffer = new ArrayBuffer(0, { maxByteLength: maxSize }); + let view = new Uint8Array(buffer); /** * @param {number} byteLength */ const readUntil = async (byteLength) => { + assert(byteLength <= buffer.maxByteLength); if (buffer.byteLength >= byteLength) return; const reader = stream.getReader(); + let offset = buffer.byteLength; try { - const parts = [buffer]; - let size = buffer.byteLength; - while (size < byteLength) { + while (offset < byteLength) { const { value, done } = await reader.read(); if (done) throw new Error("Unexpected end of stream"); - parts.push(value); size += value.byteLength; } - buffer = concatUint8Array(parts); } finally { reader.releaseLock(); } + buffer = buffer.transfer(size); + view = new Uint8Array(buffer); + for (let j = 0; j < i; j++) { + const part = parts[j]; + view.set(part, offset); + offset += part.byteLength; + } } await readUntil(FILE_HEADER_SIZE); @@ -88,19 +92,19 @@ export async function from(stream, maxSize = 1024 * 1024) { const size = blockCount * header.blockSize; if (type === ChunkType.Raw) { - buffer = buffer.slice(CHUNK_HEADER_SIZE); - let readBytes = 0; - while (readBytes < size) { - const dataChunkSize = Math.min(size - readBytes, maxSize); - await readUntil(dataChunkSize); - const data = buffer.slice(0, dataChunkSize); + let readBytes = CHUNK_HEADER_SIZE; + while (readBytes < totalBytes) { + const dataChunkSize = Math.min(totalBytes - readBytes, maxSize); + await readUntil(readBytes + dataChunkSize); // TODO: maybe read smaller chunks? + const data = buffer.subarray(readBytes, readBytes + dataChunkSize); assert(data.byteLength === dataChunkSize); yield [offset, data, dataChunkSize]; - buffer = buffer.slice(dataChunkSize); + // buffer = buffer.slice(dataChunkSize); readBytes += dataChunkSize; offset += dataChunkSize; } assert(readBytes === size); + buffer = buffer.slice(totalBytes); } else if (type === ChunkType.Fill) { await readUntil(totalBytes); const data = buffer.slice(CHUNK_HEADER_SIZE, totalBytes);