From b947920b8ef0f64f182671c253f9c19d0873b8cd Mon Sep 17 00:00:00 2001 From: neko <225498830+neko782@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:31:05 +0000 Subject: [PATCH 1/2] add 7z handler --- package.json | 1 + src/handlers/index.ts | 2 + src/handlers/sevenZip.ts | 126 +++++++++++++++++++++++++++++++++++++++ vite.config.js | 4 ++ 4 files changed, 133 insertions(+) create mode 100644 src/handlers/sevenZip.ts diff --git a/package.json b/package.json index f24bda1e..c14f865c 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "vite-tsconfig-paths": "^6.0.5" }, "dependencies": { + "7z-wasm": "^1.2.0", "@bjorn3/browser_wasi_shim": "^0.4.2", "@bokuweb/zstd-wasm": "^0.0.27", "@ffmpeg/core": "^0.12.10", diff --git a/src/handlers/index.ts b/src/handlers/index.ts index 38ee48e4..8b45778c 100644 --- a/src/handlers/index.ts +++ b/src/handlers/index.ts @@ -21,6 +21,7 @@ import sqlite3Handler from "./sqlite.ts"; import vtfHandler from "./vtf.ts"; import mcMapHandler from "./mcmap.ts"; import jszipHandler from "./jszip.ts"; +import sevenZipHandler from "./sevenZip.ts"; import json5Handler from "./json5.ts"; import alsHandler from "./als.ts"; import qoaFuHandler from "./qoa-fu.ts"; @@ -89,6 +90,7 @@ try { handlers.push(new sqlite3Handler()) } catch (_) { }; try { handlers.push(new vtfHandler()) } catch (_) { }; try { handlers.push(new mcMapHandler()) } catch (_) { }; try { handlers.push(new jszipHandler()) } catch (_) { }; +try { handlers.push(new sevenZipHandler()) } catch (_) { }; try { handlers.push(new json5Handler()) } catch (_) { }; try { handlers.push(new alsHandler()) } catch (_) { }; try { handlers.push(new qoaFuHandler()) } catch (_) { }; diff --git a/src/handlers/sevenZip.ts b/src/handlers/sevenZip.ts new file mode 100644 index 00000000..6ca5524e --- /dev/null +++ b/src/handlers/sevenZip.ts @@ -0,0 +1,126 @@ +// file: 7z.ts + +import type { FileData, FileFormat, FormatHandler } from "../FormatHandler.ts"; +import CommonFormats, { Category } from "src/CommonFormats.ts"; +import SevenZip from "7z-wasm"; +import mime from "mime"; +import normalizeMimeType from "src/normalizeMimeType.ts"; + +const defaultSevenZipOptions = { + locateFile: () => "/convert/wasm/7zz.wasm" +} + +class sevenZipHandler implements FormatHandler { + + public name: string = "sevenZip"; + public supportedFormats: FileFormat[] = []; + public ready: boolean = false; + + public supportAnyInput: boolean = true; + + async init () { + this.supportedFormats = []; + + const stdout: number[] = []; + const sevenZip = await SevenZip({ + ...defaultSevenZipOptions, + stdout: (c) => { + stdout.push(c); + }, + }); + + sevenZip.callMain(["i"]); + + const text = new TextDecoder().decode(new Uint8Array(stdout)); + + // no codecs for now + const formatsText = text.match(/\n\n\nFormats:\n(.*?)\n\n/s); + if (!formatsText) throw new Error("7zz output did not have any formats"); + const formatLines = formatsText[1].split("\n"); + + // this will totally break in future 7z versions but its the only way + for (const formatLine of formatLines) { + // 7zz i gives more than 1 extension, but i dont think we will + // need those as they are mostly aliases and thats renamehandler's job. + // also we cant faithfully parse more than 1 extension because there is no + // way to know where extensions stop and weird signature stuff begins + const [flags, name, extension, ...extra] = formatLine.trim().split(/ +/); + + if (name === "Hash") continue; + // 7z doesnt handle tar or tar attached formats well + if (extension === "tar" || extra.includes("(.tar)")) continue; + + const mimeType = normalizeMimeType(mime.getType(extension) || `application/${extension}`); + this.supportedFormats.push({ + name: `${name} Archive`, // we cant really do better than that + format: extension, + extension, + mime: mimeType, + from: true, + to: flags.includes("C"), + internal: name, + category: Category.ARCHIVE, + lossless: false // archive metadata is too complicated + }); + } + + // quick hack to avoid big penalty for zip being down the list + const zipIndex = this.supportedFormats.findIndex(format => format.internal === "zip"); + if (zipIndex === -1) throw new Error("7z does not have zip format?"); + const zip = this.supportedFormats.splice(zipIndex, 1); + this.supportedFormats.unshift(...zip); + + this.ready = true; + } + + async doConvert ( + inputFiles: FileData[], + inputFormat: FileFormat, + outputFormat: FileFormat + ): Promise { + const outputFiles: FileData[] = []; + + if (!this.supportedFormats.some(format => format.to && format.internal === outputFormat.internal)) { + throw new Error(`sevenZipHandler cannot convert to ${outputFormat.mime}`); + } + + if (this.supportedFormats.some(format => format.internal === inputFormat.internal)) { + for (const inputFile of inputFiles) { + const sevenZip = await SevenZip(defaultSevenZipOptions); + + sevenZip.FS.writeFile(inputFile.name, inputFile.bytes); + sevenZip.callMain(["x", inputFile.name, `-odata`]); + + const name = inputFile.name.replace(/\.[^.]+$/, "") + `.${outputFormat.extension}`; + sevenZip.FS.chdir("data"); // we need to preserve the structure of the input archive + sevenZip.callMain(["a", "../" + name]); + sevenZip.FS.chdir(".."); + + const bytes = sevenZip.FS.readFile(name); + outputFiles.push({ bytes, name }); + } + } else { + const sevenZip = await SevenZip(defaultSevenZipOptions); + + sevenZip.FS.mkdir("data"); + sevenZip.FS.chdir("data"); + for (const inputFile of inputFiles) { + sevenZip.FS.writeFile(inputFile.name, inputFile.bytes); + } + + const name = inputFiles.length === 1 ? + inputFiles[0].name + `.${outputFormat.extension}` + : `archive.${outputFormat.extension}`; + sevenZip.callMain(["a", "../" + name]); + sevenZip.FS.chdir(".."); + + const bytes = sevenZip.FS.readFile(name); + outputFiles.push({ bytes, name }); + } + + return outputFiles; + } + +} + +export default sevenZipHandler; diff --git a/vite.config.js b/vite.config.js index 87bab18f..33a63133 100644 --- a/vite.config.js +++ b/vite.config.js @@ -66,6 +66,10 @@ export default defineConfig({ src: "src/handlers/tarCompressed/liblzma.wasm", dest: "wasm" }, + { + src: "node_modules/7z-wasm/7zz.wasm", + dest: "wasm" + } ] }), tsconfigPaths() From 698a134d88e82f908168f49c8dbb56aa00ca17c7 Mon Sep 17 00:00:00 2001 From: neko <225498830+neko782@users.noreply.github.com> Date: Thu, 19 Mar 2026 21:49:39 +0000 Subject: [PATCH 2/2] replaced some of the handlers with 7z --- src/handlers/index.ts | 8 --- src/handlers/jszip.ts | 41 ----------- src/handlers/sevenZip.ts | 70 ++++++++++++++++--- src/handlers/tar.ts | 126 ---------------------------------- src/handlers/tarCompressed.ts | 125 --------------------------------- src/normalizeMimeType.ts | 1 + 6 files changed, 60 insertions(+), 311 deletions(-) delete mode 100644 src/handlers/jszip.ts delete mode 100644 src/handlers/tar.ts delete mode 100644 src/handlers/tarCompressed.ts diff --git a/src/handlers/index.ts b/src/handlers/index.ts index 8b45778c..b0fbc627 100644 --- a/src/handlers/index.ts +++ b/src/handlers/index.ts @@ -20,7 +20,6 @@ import threejsHandler from "./threejs.ts"; import sqlite3Handler from "./sqlite.ts"; import vtfHandler from "./vtf.ts"; import mcMapHandler from "./mcmap.ts"; -import jszipHandler from "./jszip.ts"; import sevenZipHandler from "./sevenZip.ts"; import json5Handler from "./json5.ts"; import alsHandler from "./als.ts"; @@ -54,7 +53,6 @@ import n64romHandler from "./n64rom.ts"; import vexflowHandler from "./vexflow.ts"; import toonHandler from "./toon.ts"; import rpgmvpHandler from "./rpgmvp.ts"; -import tarHandler from "./tar.ts"; import otaHandler from "./ota.ts"; import comicsHandler from "./comics.ts"; import terrariaWldHandler from "./terrariawld.ts"; @@ -62,7 +60,6 @@ import opusMagnumHandler from "./opusMagnum.ts"; import aperturePictureHandler from "./aperturePicture.ts"; import xcfHandler from "./xcf.ts"; import pdfparseHandler from "./pdfparse.ts"; -import { tarGzHandler, tarZstdHandler, tarXzHandler } from "./tarCompressed.ts"; import mclangHandler from "./minecraftLangfileHandler.ts"; import cybergrindHandler from "./cybergrindHandler.ts"; import textToSourceHandler from "./textToSource.ts"; @@ -89,7 +86,6 @@ try { handlers.push(new threejsHandler()) } catch (_) { }; try { handlers.push(new sqlite3Handler()) } catch (_) { }; try { handlers.push(new vtfHandler()) } catch (_) { }; try { handlers.push(new mcMapHandler()) } catch (_) { }; -try { handlers.push(new jszipHandler()) } catch (_) { }; try { handlers.push(new sevenZipHandler()) } catch (_) { }; try { handlers.push(new json5Handler()) } catch (_) { }; try { handlers.push(new alsHandler()) } catch (_) { }; @@ -126,7 +122,6 @@ try { handlers.push(new n64romHandler()) } catch (_) { }; try { handlers.push(new vexflowHandler()) } catch (_) { }; try { handlers.push(new toonHandler()) } catch (_) { }; try { handlers.push(new rpgmvpHandler()) } catch (_) { }; -try { handlers.push(new tarHandler()) } catch (_) { }; try { handlers.push(new otaHandler()) } catch (_) { }; try { handlers.push(new comicsHandler()) } catch (_) { }; try { handlers.push(new terrariaWldHandler()) } catch (_) { }; @@ -134,9 +129,6 @@ try { handlers.push(new opusMagnumHandler()) } catch (_) { }; try { handlers.push(new aperturePictureHandler()) } catch (_) { }; try { handlers.push(new xcfHandler()) } catch (_) { }; try { handlers.push(new pdfparseHandler()) } catch (_) { }; -try { handlers.push(tarGzHandler) } catch (_) { }; -try { handlers.push(tarZstdHandler) } catch (_) { }; -try { handlers.push(tarXzHandler) } catch (_) { }; try { handlers.push(new mclangHandler()) } catch (_) { }; try { handlers.push(new cybergrindHandler()) } catch (_) { }; try { handlers.push(new textToSourceHandler()) } catch (_) { }; diff --git a/src/handlers/jszip.ts b/src/handlers/jszip.ts deleted file mode 100644 index 518f9bc6..00000000 --- a/src/handlers/jszip.ts +++ /dev/null @@ -1,41 +0,0 @@ -import CommonFormats from "src/CommonFormats.ts"; -import type { FileData, FileFormat, FormatHandler } from "../FormatHandler.ts"; -import JSZip from "jszip"; - -class jszipHandler implements FormatHandler { - - public name: string = "jszip"; - - public supportedFormats: FileFormat[] = [ - CommonFormats.ZIP.builder("zip").allowTo() - ]; - - public supportAnyInput: boolean = true; - - public ready: boolean = false; - - async init() { - this.ready = true; - } - - async doConvert( - inputFiles: FileData[], - inputFormat: FileFormat, - outputFormat: FileFormat - ): Promise { - - const outputFiles: FileData[] = []; - const zip = new JSZip(); - - for (const file of inputFiles) { - zip.file(file.name, file.bytes); - } - - const output = await zip.generateAsync({ type: "uint8array" }); - - outputFiles.push({ bytes: output, name: "output.zip" }); - return outputFiles; - } -} - -export default jszipHandler; diff --git a/src/handlers/sevenZip.ts b/src/handlers/sevenZip.ts index 6ca5524e..359b73f8 100644 --- a/src/handlers/sevenZip.ts +++ b/src/handlers/sevenZip.ts @@ -18,8 +18,11 @@ class sevenZipHandler implements FormatHandler { public supportAnyInput: boolean = true; + #tarCompressedFormats: string[] = []; + async init () { this.supportedFormats = []; + this.#tarCompressedFormats = []; const stdout: number[] = []; const sevenZip = await SevenZip({ @@ -47,28 +50,40 @@ class sevenZipHandler implements FormatHandler { const [flags, name, extension, ...extra] = formatLine.trim().split(/ +/); if (name === "Hash") continue; - // 7z doesnt handle tar or tar attached formats well - if (extension === "tar" || extra.includes("(.tar)")) continue; - const mimeType = normalizeMimeType(mime.getType(extension) || `application/${extension}`); + let displayName = `${name} archive`; + let format = extension; + + if (extra.includes("(.tar)")) { + // compressed formats will be only tar for now + this.#tarCompressedFormats.push(name); + displayName = `${name} compressed tar archive`; + format = `tar.${extension}`; + } + this.supportedFormats.push({ - name: `${name} Archive`, // we cant really do better than that - format: extension, + name: displayName, + format, extension, mime: mimeType, from: true, to: flags.includes("C"), internal: name, category: Category.ARCHIVE, - lossless: false // archive metadata is too complicated + lossless: false, // archive metadata is too complicated }); } - // quick hack to avoid big penalty for zip being down the list - const zipIndex = this.supportedFormats.findIndex(format => format.internal === "zip"); - if (zipIndex === -1) throw new Error("7z does not have zip format?"); - const zip = this.supportedFormats.splice(zipIndex, 1); - this.supportedFormats.unshift(...zip); + // push zip and tar up the list + const priority = ["tar", "zip"]; + const prioritized = []; + for (const format of priority) { + prioritized.push(this.supportedFormats.find(f => f.internal === format)!); + } + this.supportedFormats = [ + ...prioritized, + ...this.supportedFormats.filter(f => !priority.includes(f.internal)) + ]; this.ready = true; } @@ -84,6 +99,39 @@ class sevenZipHandler implements FormatHandler { throw new Error(`sevenZipHandler cannot convert to ${outputFormat.mime}`); } + // handle compressed tars + if (this.#tarCompressedFormats.includes(inputFormat.internal) + || this.#tarCompressedFormats.includes(outputFormat.internal)) { + + if (outputFormat.internal === "tar") { + for (const inputFile of inputFiles) { + const sevenZip = await SevenZip(defaultSevenZipOptions); + + sevenZip.FS.writeFile(inputFile.name, inputFile.bytes); + sevenZip.callMain(["x", inputFile.name]); + + const name = inputFile.name.replace(/\.[^.]+$/, ""); + const bytes = sevenZip.FS.readFile(name); + outputFiles.push({ bytes, name }); + } + } else if (inputFormat.internal === "tar") { + for (const inputFile of inputFiles) { + const sevenZip = await SevenZip(defaultSevenZipOptions); + sevenZip.FS.writeFile(inputFile.name, inputFile.bytes); + + const name = inputFile.name + `.${outputFormat.extension}`; + sevenZip.callMain(["a", name, inputFile.name]); + + const bytes = sevenZip.FS.readFile(name); + outputFiles.push({ bytes, name }); + } + } else { + throw new Error(`sevenZipHandler cannot convert from ${inputFormat.mime} to ${outputFormat.mime}`); + } + + return outputFiles; + } + if (this.supportedFormats.some(format => format.internal === inputFormat.internal)) { for (const inputFile of inputFiles) { const sevenZip = await SevenZip(defaultSevenZipOptions); diff --git a/src/handlers/tar.ts b/src/handlers/tar.ts deleted file mode 100644 index 39bfdc82..00000000 --- a/src/handlers/tar.ts +++ /dev/null @@ -1,126 +0,0 @@ -// file: tar.ts - -import type { FileData, FileFormat, FormatHandler } from "../FormatHandler.ts"; -import CommonFormats from "src/CommonFormats.ts"; - -import { - createTar, - parseTar, - type TarFileItem, -} from "nanotar"; -import JSZip from "jszip"; - -class tarHandler implements FormatHandler { - - public name: string = "tar"; - public supportedFormats?: FileFormat[] = [ - CommonFormats.TAR.builder("tar").allowFrom().allowTo().markLossless(), - CommonFormats.ZIP.builder("zip").allowFrom().allowTo().markLossless(), - { - name: "Comic Book Archive (ZIP)", - format: "cbz", - extension: "cbz", - mime: "application/vnd.comicbook+zip", - from: true, - to: true, - internal: "cbz", - category: ["archive"], - }, - { - name: "Comic Book Archive (TAR)", - format: "cbt", - extension: "cbt", - mime: "application/vnd.comicbook+tar", - from: true, - to: true, - internal: "cbt", - category: ["archive"], - } - ]; - - public supportAnyInput: boolean = true; - - public ready: boolean = false; - - async init() { - this.ready = true; - } - - async doConvert( - inputFiles: FileData[], - inputFormat: FileFormat, - outputFormat: FileFormat - ): Promise { - const outputFiles: FileData[] = []; - - if ((inputFormat.internal === "zip" && outputFormat.internal === "tar") || (inputFormat.internal === "cbz" && outputFormat.internal === "cbt")) { - for (const inputFile of inputFiles) { - const zip = new JSZip(); - await zip.loadAsync(inputFile.bytes); - - const archiveFiles: TarFileItem[] = []; - - for (const [filename, zipEntry] of Object.entries(zip.files)) { - if (zipEntry.dir) { - archiveFiles.push({ name: filename }); - continue; - } - const data = await zipEntry.async("uint8array"); - - archiveFiles.push({ - name: filename, - data, - attrs: { - mtime: zipEntry.date.getTime(), - mode: zipEntry.unixPermissions?.toString(8) - } - }); - } - - const name = inputFile.name.replace(/\.zip$/i, ".tar").replace(/\.cbz$/i, ".cbt"); - const bytes = createTar( - archiveFiles, - {}, - ); - - outputFiles.push({ bytes, name }); - } - } else if ((inputFormat.internal === "tar" && outputFormat.internal === "zip") || (inputFormat.internal === "cbt" && outputFormat.internal === "cbz")) { - for (const inputFile of inputFiles) { - const files = parseTar(inputFile.bytes); - - const zip = new JSZip(); - - for (const file of files) { - const date = file.attrs?.mtime ? new Date(file.attrs?.mtime * 1000) : undefined; - const unixPermissions = file.attrs?.mode; - - if (!file.data) { - zip.file(file.name, null, { dir: true, date, unixPermissions }); - } else { - zip.file(file.name, file.data, { date, unixPermissions }); - } - } - - const bytes = await zip.generateAsync({ type: "uint8array" }); - - const name = inputFile.name.replace(/\.tar$/i, ".zip").replace(/\.cbt$/i, ".cbz"); - outputFiles.push({ bytes, name }); - } - } else if (outputFormat.internal === "tar") { - const bytes = createTar( - inputFiles.map(file => ({ name: file.name, data: file.bytes })), - {}, - ); - const name = inputFiles.length === 1 ? inputFiles[0].name + ".tar" : "archive.tar"; - outputFiles.push({ bytes, name }) - } else { - throw new Error("tarHandler cannot process this conversion"); - } - - return outputFiles; - } - -} - -export default tarHandler; diff --git a/src/handlers/tarCompressed.ts b/src/handlers/tarCompressed.ts deleted file mode 100644 index c9544164..00000000 --- a/src/handlers/tarCompressed.ts +++ /dev/null @@ -1,125 +0,0 @@ -// file: tarCompressed.ts - -import type { FileData, FileFormat, FormatHandler } from "../FormatHandler.ts"; -import CommonFormats from "src/CommonFormats.ts"; -import { gzipSync as gzip, gunzipSync as gunzip } from "fflate"; -import { compress as zstd, decompress as unzstd, init as zstd_init } from "@bokuweb/zstd-wasm"; -import { compress as xz, decompress as unxz, init as xz_init } from "./tarCompressed/xz.ts"; - -function tarCompressedHandler( - name: string, - format: FileFormat, - init: () => Promise, - compress: (inputFile: FileData, outputFiles: FileData[]) => Promise, - decompress: (inputFile: FileData, outputFiles: FileData[]) => Promise, -): FormatHandler { - return { - name: name, - ready: false, - supportedFormats: [ - CommonFormats.TAR.builder("tar").allowFrom().allowTo().markLossless(), - format - ], - - async init() { - await init(); - this.ready = true; - }, - - async doConvert ( - inputFiles: FileData[], - inputFormat: FileFormat, - outputFormat: FileFormat - ): Promise { - const outputFiles: FileData[] = []; - - for (const inputFile of inputFiles) { - if (inputFormat.internal === "tar") { - await compress(inputFile, outputFiles); - } else if (outputFormat.internal === "tar") { - await decompress(inputFile, outputFiles); - } else { - throw new Error(`${name} cannot process this conversion`); - } - } - - return outputFiles; - } - }; -} - -export const tarGzHandler = tarCompressedHandler( - "tarGz", - { - name: "Gzipped Tape Archive", - format: "tar.gz", - extension: "gz", - mime: "application/gzip", - from: true, - to: true, - internal: "tar.gz", - category: "archive", - lossless: true - }, - async () => {}, - async (inputFile, outputFiles) => { - const bytes = gzip(inputFile.bytes); - outputFiles.push({ bytes, name: inputFile.name + ".gz" }); - }, - async (inputFile, outputFiles) => { - const bytes = gunzip(inputFile.bytes); - outputFiles.push({ bytes, name: inputFile.name.replace(/\.gz$/i, "") }); - }, -); - -export const tarZstdHandler = tarCompressedHandler( - "tarZstd", - { - name: "Zstd compressed Tape Archive", - format: "tar.zst", - extension: "zst", - mime: "application/zstd", - from: true, - to: true, - internal: "tar.zst", - category: "archive", - lossless: true - }, - async () => { - await zstd_init(); - }, - async (inputFile, outputFiles) => { - const bytes = zstd(inputFile.bytes); - outputFiles.push({ bytes, name: inputFile.name + ".zst" }); - }, - async (inputFile, outputFiles) => { - const bytes = unzstd(inputFile.bytes); - outputFiles.push({ bytes, name: inputFile.name.replace(/\.zst$/i, "") }); - }, -); - -export const tarXzHandler = tarCompressedHandler( - "tarXz", - { - name: "XZ compressed Tape Archive", - format: "tar.xz", - extension: "xz", - mime: "application/x-xz", - from: true, - to: true, - internal: "tar.xz", - category: "archive", - lossless: true - }, - async () => { - await xz_init(); - }, - async (inputFile, outputFiles) => { - const bytes = xz(inputFile.bytes); - outputFiles.push({ bytes, name: inputFile.name + ".xz" }); - }, - async (inputFile, outputFiles) => { - const bytes = unxz(inputFile.bytes); - outputFiles.push({ bytes, name: inputFile.name.replace(/\.xz$/i, "") }); - }, -); \ No newline at end of file diff --git a/src/normalizeMimeType.ts b/src/normalizeMimeType.ts index a0c700fc..ace45f4f 100644 --- a/src/normalizeMimeType.ts +++ b/src/normalizeMimeType.ts @@ -4,6 +4,7 @@ function normalizeMimeType (mime: string) { case "audio/vnd.wave": return "audio/wav"; case "application/ogg": return "audio/ogg"; case "application/x-gzip": return "application/gzip"; + case "application/zst": return "application/zstd"; case "application/x-zstd": return "application/zstd"; case "image/x-icns": return "image/icns"; case "image/x-icon": return "image/vnd.microsoft.icon";