From 7fbe0f20fe168b4878ce14b3dae170ced3956e99 Mon Sep 17 00:00:00 2001 From: smowden Date: Mon, 12 May 2025 17:12:23 +0200 Subject: [PATCH 1/5] feat(loader): add unsafe mode --- loader/src/index.cjs | 299 ++++++++++++++++++++++++------------------- package.json | 3 +- 2 files changed, 168 insertions(+), 134 deletions(-) diff --git a/loader/src/index.cjs b/loader/src/index.cjs index 34404cdc1..a3f45ae06 100644 --- a/loader/src/index.cjs +++ b/loader/src/index.cjs @@ -1,9 +1,12 @@ +let __DISABLE_METERING_FOR_LOADER = false; // Global flag for disabling metering + const Emscripten = require('./formats/emscripten.cjs') const Emscripten2 = require('./formats/emscripten2.cjs') const Emscripten3 = require('./formats/emscripten3.cjs') const Emscripten4 = require('./formats/emscripten4.cjs') const Wasm64 = require('./formats/wasm64-emscripten.cjs') -const metering = require('@permaweb/wasm-metering') +const metering = require('@permaweb/wasm-metering') // Re-enabled metering +// const metering = require('@permaweb/wasm-metering') /* eslint-enable */ @@ -110,168 +113,198 @@ const metering = require('@permaweb/wasm-metering') const originalCompileStreaming = WebAssembly.compileStreaming const originalCompile = WebAssembly.compile +// Updated shouldApplyMetering to respect __DISABLE_METERING_FOR_LOADER and importObject.format const shouldApplyMetering = (importObject = {}) => { + if (__DISABLE_METERING_FOR_LOADER) return false; // If metering is globally disabled for this call, never meter + // Original logic: meter only if format explicitly asks for it return ['wasm32-unknown-emscripten-metering', 'wasm64-unknown-emscripten-draft_2024_10_16-metering'].includes(importObject.format); -} +}; +// Re-enabled and corrected applyMetering function const applyMetering = (arrayBuffer, importObject) => { - const { format } = importObject + const { format } = importObject; // This format is key. const view = ArrayBuffer.isView(arrayBuffer) ? arrayBuffer - : Buffer.from(arrayBuffer) + : Buffer.from(arrayBuffer); // Determine meter type and apply metering - const meterType = format.startsWith('wasm32') ? 'i32' : 'i64' - const meteredView = metering.meterWASM(view, { meterType }) + const meterType = format.startsWith('wasm32') ? 'i32' : 'i64'; + const meteredView = metering.meterWASM(view, { meterType }); - return originalCompile(meteredView.buffer) -} + return originalCompile(meteredView.buffer); // Use originalCompile (WebAssembly.compile) +}; -// Override WebAssembly.compileStreaming to apply metering +// Override WebAssembly.compileStreaming to apply metering conditionally WebAssembly.compileStreaming = async function (source, importObject = {}) { - if (!shouldApplyMetering(importObject)) return originalCompileStreaming(source) + if (!shouldApplyMetering(importObject)) return originalCompileStreaming(source); - const arrayBuffer = await source.arrayBuffer() - return applyMetering(arrayBuffer, importObject) -} + const arrayBuffer = await source.arrayBuffer(); + return applyMetering(arrayBuffer, importObject); +}; -// Override WebAssembly.compile to apply metering +// Override WebAssembly.compile to apply metering conditionally WebAssembly.compile = async function (source, importObject = {}) { - if (!shouldApplyMetering(importObject)) return originalCompile(source) - - return applyMetering(source, importObject) -} + if (!shouldApplyMetering(importObject)) return originalCompile(source); + // source is already an ArrayBuffer or WebAssembly.Module here + return applyMetering(source, importObject); +}; module.exports = async function (binary, options) { - let instance = null - let doHandle = null + if (options === null || typeof options === 'undefined') { + options = { format: 'wasm32-unknown-emscripten' }; + } else if (typeof options !== 'object') { + options = { format: 'wasm32-unknown-emscripten' }; + } - let meterType = options.format.startsWith("wasm32") ? "i32" : "i64"; + const disableMetering = options.DISABLE_METERING === true; + const unsafeMemory = options.UNSAFE_MEMORY === true; + const originalGlobalDisableMeteringFlag = __DISABLE_METERING_FOR_LOADER; + __DISABLE_METERING_FOR_LOADER = disableMetering; // Set global flag based on option - if (options === null) { - options = { format: 'wasm32-unknown-emscripten' } - } - if (options.format === "wasm32-unknown-emscripten") { - instance = await Emscripten(binary, options) - } else if (options.format === "wasm32-unknown-emscripten2") { - instance = await Emscripten2(binary, options) - } else if (options.format === "wasm32-unknown-emscripten3") { - instance = await Emscripten3(binary, options) - } else { - - if (typeof binary === "function") { - options.instantiateWasm = binary - } else { - if (options.format === "wasm32-unknown-emscripten-metering" || options.format === "wasm64-unknown-emscripten-draft_2024_10_16-metering") { - binary = metering.meterWASM(binary, { meterType }) - } - options.wasmBinary = binary - } + try { + let instance = null; + let doHandle = null; + let meterType; - if (options.format === "wasm64-unknown-emscripten-draft_2024_02_15" || options.format === "wasm64-unknown-emscripten-draft_2024_10_16-metering") { - instance = await Wasm64(options) - } else if (options.format === "wasm32-unknown-emscripten4" || options.format === "wasm32-unknown-emscripten-metering") { - instance = await Emscripten4(options) + if (!disableMetering) { // meterType is only needed if metering is not disabled + const currentFormat = options.format || 'wasm32-unknown-emscripten'; + meterType = currentFormat.startsWith("wasm32") ? "i32" : "i64"; } - await instance['FS_createPath']('/', 'data') - - doHandle = instance.cwrap('handle', 'string', ['string', 'string'], { async: true }) - } - - /** - * Since the module can be invoked multiple times, there isn't really - * a good place to cleanup these listeners (since emscripten doesn't do it), - * other than immediately. - * - * I don't really see where they are used, since CU implementations MUST - * catch reject Promises from the WASM module, as part of evaluation. - * - * TODO: maybe a better way to do this - * - * So we immediately remove any listeners added by Module, - * in order to prevent memory leaks - */ - if (instance.cleanupListeners) { - instance.cleanupListeners() - } + if (options.format === "wasm32-unknown-emscripten") { + instance = await Emscripten(binary, options) + } else if (options.format === "wasm32-unknown-emscripten2") { + instance = await Emscripten2(binary, options) + } else if (options.format === "wasm32-unknown-emscripten3") { + instance = await Emscripten3(binary, options) + } else { + if (typeof binary === "function") { + options.instantiateWasm = binary + } else { + // Local metering for specific formats, only if metering is NOT disabled + if (!disableMetering && (options.format === "wasm32-unknown-emscripten-metering" || options.format === "wasm64-unknown-emscripten-draft_2024_10_16-metering")) { + if (metering && meterType) { + binary = metering.meterWASM(binary, { meterType }); + } + } + options.wasmBinary = binary + } - if (options.format !== "wasm64-unknown-emscripten-draft_2024_02_15" && - options.format !== "wasm32-unknown-emscripten4" && - options.format !== "wasm32-unknown-emscripten-metering" && - options.format !== "wasm64-unknown-emscripten-draft_2024_10_16-metering") { - doHandle = instance.cwrap('handle', 'string', ['string', 'string']) - } + if (options.format === "wasm64-unknown-emscripten-draft_2024_02_15" || options.format === "wasm64-unknown-emscripten-draft_2024_10_16-metering") { + instance = await Wasm64(options) + } else if (options.format === "wasm32-unknown-emscripten4" || options.format === "wasm32-unknown-emscripten-metering") { + instance = await Emscripten4(options) + } + await instance['FS_createPath']('/', 'data') + doHandle = instance.cwrap('handle', 'string', ['string', 'string'], { async: true }) + } - return async (buffer, msg, env) => { /** - * instance is the Module - * abstraction for the WebAssembly environnment - * - * This is where things like module options, extensions - * and other metadata are exposed to things like WeaveDrive. - * - * So by setting blockHeight on the instance to the current - * msg's blockHeight, it can be made available to WeaveDrive - * for admissable checks + * Since the module can be invoked multiple times, there isn't really + * a good place to cleanup these listeners (since emscripten doesn't do it), + * other than immediately. + * + * I don't really see where they are used, since CU implementations MUST + * catch reject Promises from the WASM module, as part of evaluation. + * + * TODO: maybe a better way to do this + * + * So we immediately remove any listeners added by Module, + * in order to prevent memory leaks */ - instance.blockHeight = msg['Block-Height'] - - const originalRandom = Math.random - // const OriginalDate = Date - const originalLog = console.log - try { - /** start mock Math.random */ - Math.random = function () { return 0.5 } - /** end mock Math.random */ - - /** start mock console.log */ - console.log = function () { return null } - /** end mock console.log */ - - if (buffer) { - if (instance.HEAPU8.byteLength < buffer.byteLength) { - console.log("RESIZE HEAP") - instance.resizeHeap(buffer.byteLength) - } - instance.HEAPU8.set(buffer) + if (instance.cleanupListeners) { + instance.cleanupListeners() + } + + if (options.format !== "wasm64-unknown-emscripten-draft_2024_02_15" && + options.format !== "wasm32-unknown-emscripten4" && + options.format !== "wasm32-unknown-emscripten-metering" && + options.format !== "wasm64-unknown-emscripten-draft_2024_10_16-metering") { + // Ensure doHandle is assigned if not in the async:true path from above + if (!doHandle) { + doHandle = instance.cwrap('handle', 'string', ['string', 'string']) } + } + + return async (buffer, msg, env) => { /** - * Make sure to refill the gas tank for each invocation + * instance is the Module + * abstraction for the WebAssembly environnment + * + * This is where things like module options, extensions + * and other metadata are exposed to things like WeaveDrive. + * + * So by setting blockHeight on the instance to the current + * msg's blockHeight, it can be made available to WeaveDrive + * for admissable checks */ - instance.gas.refill() - - const res = await doHandle(JSON.stringify(msg), JSON.stringify(env)) - - const { ok, response } = JSON.parse(res) - if (!ok) throw response - - /** unmock functions */ - // eslint-disable-next-line no-global-assign - // Date = OriginalDate - Math.random = originalRandom - console.log = originalLog - /** end unmock */ - - return { - Memory: instance.HEAPU8.slice(), - Error: response.Error, - Output: response.Output, - Messages: response.Messages, - Spawns: response.Spawns, - Assignments: response.Assignments, - Patches: response.Patches || [], - GasUsed: instance.gas.used + instance.blockHeight = msg['Block-Height'] + + const originalRandom = Math.random + // const OriginalDate = Date + const originalLog = console.log + try { + /** start mock Math.random */ + Math.random = function () { return 0.5 } + /** end mock Math.random */ + + /** start mock console.log */ + console.log = function () { return null } + /** end mock console.log */ + + if (buffer) { + if (instance.HEAPU8.byteLength < buffer.byteLength) { + console.log("RESIZE HEAP") + instance.resizeHeap(buffer.byteLength) + } + instance.HEAPU8.set(buffer) + } + /** + * Make sure to refill the gas tank for each invocation, if metering is not disabled + */ + if (!disableMetering && instance.gas && typeof instance.gas.refill === 'function') { + instance.gas.refill(); + } + + const res = await doHandle(JSON.stringify(msg), JSON.stringify(env)) + + const { ok, response } = JSON.parse(res) + if (!ok) throw response + + /** unmock functions */ + // eslint-disable-next-line no-global-assign + // Date = OriginalDate + Math.random = originalRandom + console.log = originalLog + /** end unmock */ + + const result = { + Memory: unsafeMemory ? instance.HEAPU8.buffer : instance.HEAPU8.slice(), // Conditional memory return + Error: response.Error, + Output: response.Output, + Messages: response.Messages, + Spawns: response.Spawns, + Assignments: response.Assignments, + Patches: response.Patches || [], + }; + + // Add GasUsed only if metering is not disabled + if (!disableMetering && instance.gas && typeof instance.gas.used !== 'undefined') { + result.GasUsed = instance.gas.used; + } + return result; + } finally { + // eslint-disable-next-line no-global-assign + // Date = OriginalDate + Math.random = originalRandom + console.log = originalLog + buffer = null } - } finally { - // eslint-disable-next-line no-global-assign - // Date = OriginalDate - Math.random = originalRandom - console.log = originalLog - buffer = null - } + }; + } finally { + __DISABLE_METERING_FOR_LOADER = originalGlobalDisableMeteringFlag; // Restore global flag } -} +}; diff --git a/package.json b/package.json index dc025e624..a322dfe3d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "@permaweb/ao", "description": "ao monorepo", + "type": "module", "workspaces": [], "scripts": { "fmt": "standard --fix", @@ -24,4 +25,4 @@ "**/contract.js" ] } -} +} \ No newline at end of file From 796c546d3aea6fe3cd3b6ab742d157e27c56a3ad Mon Sep 17 00:00:00 2001 From: smowden Date: Mon, 12 May 2025 17:40:24 +0200 Subject: [PATCH 2/5] chore(loader): make code more concise --- loader/src/index.cjs | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/loader/src/index.cjs b/loader/src/index.cjs index a3f45ae06..cb8c008af 100644 --- a/loader/src/index.cjs +++ b/loader/src/index.cjs @@ -1,5 +1,3 @@ -let __DISABLE_METERING_FOR_LOADER = false; // Global flag for disabling metering - const Emscripten = require('./formats/emscripten.cjs') const Emscripten2 = require('./formats/emscripten2.cjs') const Emscripten3 = require('./formats/emscripten3.cjs') @@ -113,9 +111,8 @@ const metering = require('@permaweb/wasm-metering') // Re-enabled metering const originalCompileStreaming = WebAssembly.compileStreaming const originalCompile = WebAssembly.compile -// Updated shouldApplyMetering to respect __DISABLE_METERING_FOR_LOADER and importObject.format +// Updated shouldApplyMetering to respect importObject.format only const shouldApplyMetering = (importObject = {}) => { - if (__DISABLE_METERING_FOR_LOADER) return false; // If metering is globally disabled for this call, never meter // Original logic: meter only if format explicitly asks for it return ['wasm32-unknown-emscripten-metering', 'wasm64-unknown-emscripten-draft_2024_10_16-metering'].includes(importObject.format); }; @@ -157,24 +154,22 @@ module.exports = async function (binary, options) { options = { format: 'wasm32-unknown-emscripten' }; } - const disableMetering = options.DISABLE_METERING === true; const unsafeMemory = options.UNSAFE_MEMORY === true; - const originalGlobalDisableMeteringFlag = __DISABLE_METERING_FOR_LOADER; - __DISABLE_METERING_FOR_LOADER = disableMetering; // Set global flag based on option - try { let instance = null; let doHandle = null; let meterType; - if (!disableMetering) { // meterType is only needed if metering is not disabled - const currentFormat = options.format || 'wasm32-unknown-emscripten'; + // Determine meterType if a metering format is potentially used + const currentFormat = options.format || 'wasm32-unknown-emscripten'; + if (shouldApplyMetering({ format: currentFormat })) { meterType = currentFormat.startsWith("wasm32") ? "i32" : "i64"; } if (options.format === "wasm32-unknown-emscripten") { instance = await Emscripten(binary, options) + options.instantiateWasm = binary } else if (options.format === "wasm32-unknown-emscripten2") { instance = await Emscripten2(binary, options) } else if (options.format === "wasm32-unknown-emscripten3") { @@ -183,8 +178,8 @@ module.exports = async function (binary, options) { if (typeof binary === "function") { options.instantiateWasm = binary } else { - // Local metering for specific formats, only if metering is NOT disabled - if (!disableMetering && (options.format === "wasm32-unknown-emscripten-metering" || options.format === "wasm64-unknown-emscripten-draft_2024_10_16-metering")) { + // Local metering for specific formats + if (options.format === "wasm32-unknown-emscripten-metering" || options.format === "wasm64-unknown-emscripten-draft_2024_10_16-metering") { if (metering && meterType) { binary = metering.meterWASM(binary, { meterType }); } @@ -263,9 +258,9 @@ module.exports = async function (binary, options) { instance.HEAPU8.set(buffer) } /** - * Make sure to refill the gas tank for each invocation, if metering is not disabled + * Make sure to refill the gas tank for each invocation */ - if (!disableMetering && instance.gas && typeof instance.gas.refill === 'function') { + if (instance.gas && typeof instance.gas.refill === 'function') { instance.gas.refill(); } @@ -291,20 +286,23 @@ module.exports = async function (binary, options) { Patches: response.Patches || [], }; - // Add GasUsed only if metering is not disabled - if (!disableMetering && instance.gas && typeof instance.gas.used !== 'undefined') { + // Add GasUsed if metering was applied + if (instance.gas && typeof instance.gas.used !== 'undefined') { result.GasUsed = instance.gas.used; } - return result; - } finally { + + /** unmock functions */ // eslint-disable-next-line no-global-assign // Date = OriginalDate Math.random = originalRandom console.log = originalLog - buffer = null - } + /** end unmock */ + buffer = null // Clear buffer reference + + return result; + } // End of try block + finally { } }; - } finally { - __DISABLE_METERING_FOR_LOADER = originalGlobalDisableMeteringFlag; // Restore global flag - } + } // End of outer try block + finally { } }; From 44bc1219aedaa668d47cc6378ecd4ca1267c2a68 Mon Sep 17 00:00:00 2001 From: smowden Date: Mon, 12 May 2025 18:06:52 +0200 Subject: [PATCH 3/5] chore(loader): remove some comments --- loader/src/index.cjs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/loader/src/index.cjs b/loader/src/index.cjs index cb8c008af..6b8e05ed3 100644 --- a/loader/src/index.cjs +++ b/loader/src/index.cjs @@ -110,16 +110,12 @@ const metering = require('@permaweb/wasm-metering') // Re-enabled metering // Save the original WebAssembly.compile and compileStreaming functions const originalCompileStreaming = WebAssembly.compileStreaming const originalCompile = WebAssembly.compile - -// Updated shouldApplyMetering to respect importObject.format only const shouldApplyMetering = (importObject = {}) => { - // Original logic: meter only if format explicitly asks for it return ['wasm32-unknown-emscripten-metering', 'wasm64-unknown-emscripten-draft_2024_10_16-metering'].includes(importObject.format); }; -// Re-enabled and corrected applyMetering function const applyMetering = (arrayBuffer, importObject) => { - const { format } = importObject; // This format is key. + const { format } = importObject; const view = ArrayBuffer.isView(arrayBuffer) ? arrayBuffer @@ -143,7 +139,6 @@ WebAssembly.compileStreaming = async function (source, importObject = {}) { // Override WebAssembly.compile to apply metering conditionally WebAssembly.compile = async function (source, importObject = {}) { if (!shouldApplyMetering(importObject)) return originalCompile(source); - // source is already an ArrayBuffer or WebAssembly.Module here return applyMetering(source, importObject); }; From 2631eff8e5529f98793194d6dac1d1f76245d91c Mon Sep 17 00:00:00 2001 From: smowden Date: Tue, 13 May 2025 11:41:34 +0200 Subject: [PATCH 4/5] chore(loader): add unsafe mode to the test suite --- loader/package.json | 3 +-- loader/test/loader.test.js | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/loader/package.json b/loader/package.json index 032314c14..f1ac93049 100644 --- a/loader/package.json +++ b/loader/package.json @@ -7,7 +7,6 @@ "directory": "loader" }, "sideEffects": false, - "type": "module", "main": "./dist/index.cjs", "types": "./dist/index.d.cts", "files": [ @@ -31,4 +30,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/loader/test/loader.test.js b/loader/test/loader.test.js index eb1022c98..52012b0d0 100644 --- a/loader/test/loader.test.js +++ b/loader/test/loader.test.js @@ -26,20 +26,40 @@ const testCases = [ binaryPath: './test/emscripten4/process.wasm', // Path to the 32-bit WASM binary options: { format: 'wasm32-unknown-emscripten4' } // Format for 32-bit }, + { + name: 'Emscripten4 (32-bit) Unsafe', + binaryPath: './test/emscripten4/process.wasm', + options: { format: 'wasm32-unknown-emscripten4', UNSAFE_MEMORY: true } + }, { name: 'Wasm64-Emscripten (64-bit)', // Test for the 64-bit WASM binaryPath: './test/wasm64-emscripten/process.wasm', // Path to the 64-bit WASM binary options: { format: 'wasm64-unknown-emscripten-draft_2024_02_15' } // Format for 64-bit }, + { + name: 'Wasm64-Emscripten (64-bit) Unsafe', + binaryPath: './test/wasm64-emscripten/process.wasm', + options: { format: 'wasm64-unknown-emscripten-draft_2024_02_15', UNSAFE_MEMORY: true } + }, { name: 'Emscripten4 Metering (32-bit)', // Test for the 32-bit WASM binaryPath: './test/emscripten4/process.wasm', // Path to the 32-bit WASM binary options: { format: 'wasm32-unknown-emscripten-metering' } // Format for 32-bit metering }, + { + name: 'Emscripten4 Metering (32-bit) Unsafe', + binaryPath: './test/emscripten4/process.wasm', + options: { format: 'wasm32-unknown-emscripten-metering', UNSAFE_MEMORY: true } + }, { name: 'Wasm64-Emscripten Metering (64-bit)', // Test for the 64-bit WASM binaryPath: './test/wasm64-emscripten/process.wasm', // Path to the 64-bit WASM binary options: { format: 'wasm64-unknown-emscripten-draft_2024_10_16-metering' } // Format for 64-bit metering + }, + { + name: 'Wasm64-Emscripten Metering (64-bit) Unsafe', + binaryPath: './test/wasm64-emscripten/process.wasm', + options: { format: 'wasm64-unknown-emscripten-draft_2024_10_16-metering', UNSAFE_MEMORY: true } } ] @@ -79,7 +99,7 @@ describe('AoLoader Functionality Tests', () => { it('load wasm and evaluate message', async () => { const handle = await AoLoader(wasmBinary, options) const mainResult = await handle(null, getMsg('return \'Hello World\''), getEnv()) - //const mainResult = await handle(null, getMsg("print(os.getenv(\"PATH\"))"),getEnv()) + // const mainResult = await handle(null, getMsg("print(os.getenv(\"PATH\"))"),getEnv()) // Check basic properties of the result assert.ok(mainResult.Memory) assert.ok(mainResult.hasOwnProperty('Messages')) From f80fe27b644b48bf7d5a0ed739650821949b6560 Mon Sep 17 00:00:00 2001 From: smowden Date: Tue, 13 May 2025 11:56:16 +0200 Subject: [PATCH 5/5] chore(loader): remove module --- loader/package.json | 1 + package.json | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/package.json b/loader/package.json index f1ac93049..665f93b16 100644 --- a/loader/package.json +++ b/loader/package.json @@ -7,6 +7,7 @@ "directory": "loader" }, "sideEffects": false, + "type": "module", "main": "./dist/index.cjs", "types": "./dist/index.d.cts", "files": [ diff --git a/package.json b/package.json index a322dfe3d..dc025e624 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { "name": "@permaweb/ao", "description": "ao monorepo", - "type": "module", "workspaces": [], "scripts": { "fmt": "standard --fix", @@ -25,4 +24,4 @@ "**/contract.js" ] } -} \ No newline at end of file +}