EIP-4844: Add c-kzg library and current trusted setup

.gitignore

@@ -50,6 +50,9 @@ docs/reference/cli.md
 *.ssz
 .pyrmont
 .tmpdb
+# EIP-4844 (only commit .ssz file)
+packages/beacon-node/trusted_setup.json
+packages/beacon-node/trusted_setup.txt
 
 # Wallet CLI artifacts
 .pass

packages/beacon-node/package.json

@@ -98,6 +98,7 @@
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {
+    "@chainsafe/as-chacha20poly1305": "^0.1.0",
     "@chainsafe/as-sha256": "^0.3.1",
     "@chainsafe/bls": "7.1.1",
     "@chainsafe/discv5": "^1.4.0",
@@ -106,7 +107,6 @@
     "@chainsafe/persistent-merkle-tree": "^0.4.2",
     "@chainsafe/snappy-stream": "5.1.1",
     "@chainsafe/ssz": "^0.9.2",
-    "@chainsafe/as-chacha20poly1305": "^0.1.0",
     "@chainsafe/threads": "^1.10.0",
     "@ethersproject/abi": "^5.0.0",
     "@libp2p/bootstrap": "^2.0.0",
@@ -124,14 +124,15 @@
     "@lodestar/fork-choice": "^1.2.1",
     "@lodestar/light-client": "^1.2.1",
     "@lodestar/params": "^1.2.1",
+    "@lodestar/reqresp": "^1.2.1",
     "@lodestar/state-transition": "^1.2.1",
     "@lodestar/types": "^1.2.1",
     "@lodestar/utils": "^1.2.1",
     "@lodestar/validator": "^1.2.1",
-    "@lodestar/reqresp": "^1.2.1",
     "@multiformats/multiaddr": "^11.0.0",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
+    "c-kzg": "^1.0.7",
     "cross-fetch": "^3.1.4",
     "datastore-core": "^8.0.1",
     "datastore-level": "^9.0.1",

packages/beacon-node/src/node/nodejs.ts

@@ -18,6 +18,7 @@ import {createMetrics, IMetrics, HttpMetricsServer} from "../metrics/index.js";
 import {getApi, BeaconRestApiServer} from "../api/index.js";
 import {initializeExecutionEngine, initializeExecutionBuilder} from "../execution/index.js";
 import {initializeEth1ForBlockProduction} from "../eth1/index.js";
+import {loadEthereumTrustedSetup} from "../util/kzg.js";
 import {createLibp2pMetrics} from "../metrics/metrics/libp2p.js";
 import {IBeaconNodeOptions} from "./options.js";
 import {runNodeNotifier} from "./notifier.js";
@@ -141,6 +142,11 @@ export class BeaconNode {
     setMaxListeners(Infinity, controller.signal);
     const signal = controller.signal;
 
+    // TODO EIP-4844, where is the best place to do this?
+    if (config.EIP4844_FORK_EPOCH < Infinity) {
+      loadEthereumTrustedSetup();
+    }
+
     // start db if not already started
     await db.start();
 

packages/beacon-node/src/util/kzg.ts

@@ -0,0 +1,113 @@
+import path from "node:path";
+import fs from "node:fs";
+import {fileURLToPath} from "node:url";
+import {FIELD_ELEMENTS_PER_BLOB, loadTrustedSetup} from "c-kzg";
+import {fromHex, toHex} from "@lodestar/utils";
+
+// Global variable __dirname no longer available in ES6 modules.
+// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+export const TRUSTED_SETUP_BIN_FILEPATH = path.join(__dirname, "../../trusted_setup.bin");
+const TRUSTED_SETUP_JSON_FILEPATH = path.join(__dirname, "../../trusted_setup.json");
+const TRUSTED_SETUP_TXT_FILEPATH = path.join(__dirname, "../../trusted_setup.txt");
+
+const POINT_COUNT_BYTES = 4;
+const G1POINT_BYTES = 48;
+const G2POINT_BYTES = 96;
+const G1POINT_COUNT = FIELD_ELEMENTS_PER_BLOB;
+const G2POINT_COUNT = 65;
+const TOTAL_SIZE = 2 * POINT_COUNT_BYTES + G1POINT_BYTES * G1POINT_COUNT + G2POINT_BYTES * G2POINT_COUNT;
+
+/**
+ * Load our KZG trusted setup into C-KZG for later use.
+ * We persist the trusted setup as serialized bytes to save space over TXT or JSON formats.
+ * However the current c-kzg API **requires** to read from a file with a specific .txt format
+ */
+export function loadEthereumTrustedSetup(): void {
+  try {
+    const bytes = fs.readFileSync(TRUSTED_SETUP_BIN_FILEPATH);
+    const json = trustedSetupBinToJson(bytes);
+    const txt = trustedSetupJsonToTxt(json);
+    fs.writeFileSync(TRUSTED_SETUP_TXT_FILEPATH, txt);
+
+    loadTrustedSetup(TRUSTED_SETUP_TXT_FILEPATH);
+  } catch (e) {
+    (e as Error).message = `Error loading trusted setup ${TRUSTED_SETUP_JSON_FILEPATH}: ${(e as Error).message}`;
+    throw e;
+  }
+}
+
+/* eslint-disable @typescript-eslint/naming-convention */
+export interface TrustedSetupJSON {
+  setup_G1: string[];
+  setup_G2: string[];
+}
+
+type TrustedSetupBin = Uint8Array;
+type TrustedSetupTXT = string;
+
+/**
+ * Custom format defined in https://github.com/ethereum/c-kzg-4844/issues/3
+ */
+export function trustedSetupJsonToBin(data: TrustedSetupJSON): TrustedSetupBin {
+  const out = new Uint8Array(TOTAL_SIZE);
+  const dv = new DataView(out.buffer, out.byteOffset, out.byteLength);
+
+  dv.setUint32(0, G1POINT_COUNT);
+  dv.setUint32(POINT_COUNT_BYTES, G2POINT_BYTES);
+
+  for (let i = 0; i < G1POINT_COUNT; i++) {
+    const point = fromHex(data.setup_G1[i]);
+    if (point.length !== G1POINT_BYTES) throw Error(`g1 point size ${point.length} != ${G1POINT_BYTES}`);
+    out.set(point, 2 * POINT_COUNT_BYTES + i * G1POINT_BYTES);
+  }
+
+  for (let i = 0; i < G2POINT_COUNT; i++) {
+    const point = fromHex(data.setup_G2[i]);
+    if (point.length !== G2POINT_BYTES) throw Error(`g2 point size ${point.length} != ${G2POINT_BYTES}`);
+    out.set(point, 2 * POINT_COUNT_BYTES + G1POINT_COUNT * G1POINT_BYTES + i * G2POINT_BYTES);
+  }
+
+  return out;
+}
+
+export function trustedSetupBinToJson(bytes: TrustedSetupBin): TrustedSetupJSON {
+  const data: TrustedSetupJSON = {
+    setup_G1: [],
+    setup_G2: [],
+  };
+
+  if (bytes.length < TOTAL_SIZE) {
+    throw Error(`trusted_setup size ${bytes.length} < ${TOTAL_SIZE}`);
+  }
+
+  for (let i = 0; i < G1POINT_COUNT; i++) {
+    const start = 2 * POINT_COUNT_BYTES + i * G1POINT_BYTES;
+    data.setup_G1.push(toHex(bytes.slice(start, start + G1POINT_BYTES)));
+  }
+
+  for (let i = 0; i < G2POINT_COUNT; i++) {
+    const start = 2 * POINT_COUNT_BYTES + G1POINT_COUNT * G1POINT_BYTES + i * G2POINT_BYTES;
+    data.setup_G1.push(toHex(bytes.slice(start, start + G2POINT_BYTES)));
+  }
+
+  return data;
+}
+
+export function trustedSetupJsonToTxt(data: TrustedSetupJSON): TrustedSetupTXT {
+  let out = `${FIELD_ELEMENTS_PER_BLOB}\n65\n`;
+
+  for (const p of data.setup_G1) out += strip0xPrefix(p) + "\n";
+  for (const p of data.setup_G2) out += strip0xPrefix(p) + "\n";
+
+  return out;
+}
+
+function strip0xPrefix(hex: string): string {
+  if (hex.startsWith("0x")) {
+    return hex.slice(2);
+  } else {
+    return hex;
+  }
+}

packages/beacon-node/test/unit/util/kzg.test.ts

@@ -0,0 +1,42 @@
+import crypto from "node:crypto";
+import {expect} from "chai";
+import {
+  freeTrustedSetup,
+  blobToKzgCommitment,
+  computeAggregateKzgProof,
+  verifyAggregateKzgProof,
+  BYTES_PER_FIELD_ELEMENT,
+  FIELD_ELEMENTS_PER_BLOB,
+} from "c-kzg";
+import {eip4844} from "@lodestar/types";
+import {loadEthereumTrustedSetup} from "../../../src/util/kzg.js";
+
+const BLOB_BYTE_COUNT = FIELD_ELEMENTS_PER_BLOB * BYTES_PER_FIELD_ELEMENT;
+
+describe("C-KZG", () => {
+  before(async function () {
+    this.timeout(10000); // Loading trusted setup is slow
+    loadEthereumTrustedSetup();
+  });
+
+  after(() => {
+    freeTrustedSetup();
+  });
+
+  it("computes the correct commitments and aggregate proofs from blobs", () => {
+    // ====================
+    // Apply this example to the test data
+    // ====================
+    const blobs = new Array(2).fill(0).map(generateRandomBlob);
+    const commitments = blobs.map(blobToKzgCommitment);
+    const proof = computeAggregateKzgProof(blobs);
+    expect(verifyAggregateKzgProof(blobs, commitments, proof)).to.equal(true);
+  });
+});
+
+/**
+ * Generate random blob of sequential integers such that each element is < BLS_MODULUS
+ */
+function generateRandomBlob(): eip4844.Blob {
+  return new Uint8Array(crypto.randomBytes(BLOB_BYTE_COUNT));
+}

packages/beacon-node/test/utils/cliTools/kzgTrustedSetupFromJson.ts

@@ -0,0 +1,16 @@
+import fs from "node:fs";
+import {TrustedSetupJSON, trustedSetupJsonToBin, TRUSTED_SETUP_BIN_FILEPATH} from "../../../src/util/kzg.js";
+
+// CLI TOOL: Use to transform a JSON trusted setup into .ssz
+//
+// Note: Closer to EIP-4844 this tool may never be useful again,
+//       see https://github.com/ethereum/c-kzg-4844/issues/3
+
+const INPUT_FILE = process.argv[2];
+if (!INPUT_FILE) throw Error("no INPUT_FILE");
+
+const json = JSON.parse(fs.readFileSync(INPUT_FILE, "utf8")) as TrustedSetupJSON;
+
+const bytes = trustedSetupJsonToBin(json);
+
+fs.writeFileSync(TRUSTED_SETUP_BIN_FILEPATH, bytes);

yarn.lock

@@ -4813,6 +4813,13 @@ bytes@3.1.2:
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
   integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==
 
+c-kzg@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/c-kzg/-/c-kzg-1.0.7.tgz#33368cac542971792c65be203ea46b33a5b2c484"
+  integrity sha512-AfTJfjTBH7N4a7JAEXbrWtdzYrrXoe/GAQYDzqkF0HUc/aJPSkKwYhWxZdvZkGamQx5LXXO9EAab3+ZnK1oOsg==
+  dependencies:
+    node-addon-api "^5.0.0"
+
 cacache@^15.2.0:
   version "15.3.0"
   resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.3.0.tgz#dc85380fb2f556fe3dda4c719bfa0ec875a7f1eb"
@@ -9818,6 +9825,11 @@ node-addon-api@^3.2.1:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
   integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==
 
+node-addon-api@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.0.0.tgz#7d7e6f9ef89043befdb20c1989c905ebde18c501"
+  integrity sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA==
+
 node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
   version "2.6.7"
   resolved "https://registry.npmjs.org/@achingbrain/node-fetch/-/node-fetch-2.6.7.tgz"