diff --git a/packages/cli/test/sim/eip4844.test.ts b/packages/cli/test/sim/eip4844.test.ts index cb1332ac6906..901d5577308d 100644 --- a/packages/cli/test/sim/eip4844.test.ts +++ b/packages/cli/test/sim/eip4844.test.ts @@ -17,7 +17,7 @@ const additionalSlotsForTTD = activePreset.SLOTS_PER_EPOCH - 2; const runTillEpoch = 6; const syncWaitEpoch = 2; -const timeout = +const runTimeoutMs = getEstimatedTimeInSecForRun({ genesisSlotDelay: genesisSlotsDelay, secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT, @@ -58,7 +58,7 @@ env.tracker.register({ }, }); -await env.start(timeout); +await env.start({runTimeoutMs}); await connectAllNodes(env.nodes); // The `TTD` will be reach around `start of bellatrixForkEpoch + additionalSlotsForMerge` slot diff --git a/packages/cli/test/sim/endpoints.test.ts b/packages/cli/test/sim/endpoints.test.ts index 1a0597a32174..8010458862d9 100644 --- a/packages/cli/test/sim/endpoints.test.ts +++ b/packages/cli/test/sim/endpoints.test.ts @@ -13,7 +13,7 @@ const genesisSlotsDelay = 10; const altairForkEpoch = 2; const bellatrixForkEpoch = 4; const validatorCount = 2; -const timeout = +const runTimeoutMs = getEstimatedTimeInSecForRun({ genesisSlotDelay: genesisSlotsDelay, secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT, @@ -42,7 +42,7 @@ const env = SimulationEnvironment.initWithDefaults( }, ] ); -await env.start(timeout); +await env.start({runTimeoutMs}); const node = env.nodes[0].cl; await waitForSlot(2, env.nodes, {env, silent: true}); diff --git a/packages/cli/test/sim/multi_fork.test.ts b/packages/cli/test/sim/multi_fork.test.ts index c6c68bb0c06f..7a08f3d30d99 100644 --- a/packages/cli/test/sim/multi_fork.test.ts +++ b/packages/cli/test/sim/multi_fork.test.ts @@ -24,7 +24,7 @@ const additionalSlotsForTTD = activePreset.SLOTS_PER_EPOCH - 2; const runTillEpoch = 6; const syncWaitEpoch = 2; -const timeout = +const runTimeoutMs = getEstimatedTimeInSecForRun({ genesisSlotDelay: genesisSlotsDelay, secondsPerSlot: SIM_TESTS_SECONDS_PER_SLOT, @@ -75,7 +75,7 @@ env.tracker.register({ }, }); -await env.start(timeout); +await env.start({runTimeoutMs}); await connectAllNodes(env.nodes); // The `TTD` will be reach around `start of bellatrixForkEpoch + additionalSlotsForMerge` slot diff --git a/packages/cli/test/utils/simulation/EpochClock.ts b/packages/cli/test/utils/simulation/EpochClock.ts index 316bf688158a..128e47da2d60 100644 --- a/packages/cli/test/utils/simulation/EpochClock.ts +++ b/packages/cli/test/utils/simulation/EpochClock.ts @@ -65,6 +65,10 @@ export class EpochClock { return this.genesisTime + slotGenesisTimeOffset; } + msToGenesis(): number { + return this.genesisTime * 1000 - Date.now(); + } + isFirstSlotOfEpoch(slot: number): boolean { return slot % this.slotsPerEpoch === 0; } diff --git a/packages/cli/test/utils/simulation/SimulationEnvironment.ts b/packages/cli/test/utils/simulation/SimulationEnvironment.ts index a990f7dd1ffd..971319dff739 100644 --- a/packages/cli/test/utils/simulation/SimulationEnvironment.ts +++ b/packages/cli/test/utils/simulation/SimulationEnvironment.ts @@ -17,6 +17,8 @@ import { EL_ETH_BASE_PORT, EL_P2P_BASE_PORT, KEY_MANAGER_BASE_PORT, + MOCK_ETH1_GENESIS_HASH, + SHARED_JWT_SECRET, SIM_TESTS_SECONDS_PER_SLOT, } from "./constants.js"; import {generateGethNode} from "./el_clients/geth.js"; @@ -48,7 +50,9 @@ import {DockerRunner} from "./runner/DockerRunner.js"; import {SimulationTracker} from "./SimulationTracker.js"; import {getEstimatedTTD} from "./utils/index.js"; -export const SHARED_JWT_SECRET = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; +interface StartOpts { + runTimeoutMs: number; +} /* eslint-disable no-console */ @@ -136,12 +140,19 @@ export class SimulationEnvironment { return env; } - async start(timeout: number): Promise { - try { - setTimeout(async () => { - await this.stop(1, "On timeout"); - }, timeout); + async start(opts: StartOpts): Promise { + setTimeout(() => { + this.stop(1, `Sim run timedout in ${opts.runTimeoutMs} ms `).catch((e) => console.error("Error on stop", e)); + }, opts.runTimeoutMs); + + const msToGenesis = this.clock.msToGenesis(); + const startTimeout = setTimeout(() => { + this.stop(1, `Start sequence not completed before genesis, in ${msToGenesis} ms`).catch((e) => + console.error("Error on stop", e) + ); + }, msToGenesis); + try { process.on("unhandledRejection", async (reason, promise) => { console.error("Unhandled Rejection at:", promise, "reason:", reason); await this.stop(1, "Unhandled promise rejection"); @@ -169,7 +180,8 @@ export class SimulationEnvironment { const el = this.nodes[i].el; // If eth1 is mock then genesis hash would be empty - const eth1Genesis = el.provider === null ? {hash: ""} : await el.provider.getBlockByNumber(0); + const eth1Genesis = + el.provider === null ? {hash: MOCK_ETH1_GENESIS_HASH} : await el.provider.getBlockByNumber(0); if (!eth1Genesis) { throw new Error(`Eth1 genesis not found for node "${this.nodes[i].id}"`); @@ -191,19 +203,28 @@ export class SimulationEnvironment { await Promise.all(this.jobs.map((j) => j.cl.start())); - await this.externalSigner.start(); - for (const node of this.nodes) { - const remoteKeys = node.cl.remoteKeys; - this.externalSigner.addKeys(remoteKeys); - await node.cl.keyManager.importRemoteKeys( - remoteKeys.map((sk) => ({pubkey: sk.toPublicKey().toHex(), url: this.externalSigner.url})) - ); + if (this.nodes.some((node) => node.cl.keys.type === "remote")) { + console.log("Starting external signer..."); + await this.externalSigner.start(); + console.log("Started external signer"); + + for (const node of this.nodes) { + if (node.cl.keys.type === "remote") { + this.externalSigner.addKeys(node.cl.keys.secretKeys); + await node.cl.keyManager.importRemoteKeys( + node.cl.keys.secretKeys.map((sk) => ({pubkey: sk.toPublicKey().toHex(), url: this.externalSigner.url})) + ); + console.log(`Imported remote keys for node ${node.id}`); + } + } } await this.tracker.start(); await Promise.all(this.nodes.map((node) => this.tracker.track(node))); } catch (error) { - await this.stop(1, `Caused error in startup. ${(error as Error).message}`); + await this.stop(1, `Error in startup. ${(error as Error).stack}`); + } finally { + clearTimeout(startTimeout); } } @@ -249,8 +270,12 @@ export class SimulationEnvironment { const clClient = this.createCLNode(cl, { id, - remoteKeys: remote ? keys : [], - localKeys: remote ? [] : keys, + keys: + keys.length > 0 && remote + ? {type: "remote", secretKeys: keys} + : keys.length > 0 + ? {type: "local", secretKeys: keys} + : {type: "no-keys"}, engineMock: typeof el === "string" ? el === ELClient.Mock : el.type === ELClient.Mock, }); @@ -264,7 +289,7 @@ export class SimulationEnvironment { private createCLNode( client: C | {type: C; options: CLClientsOptions[C]}, - options?: AtLeast + options?: AtLeast ): {job: Job; node: CLNode} { const clientType = typeof client === "object" ? client.type : client; const clientOptions = typeof client === "object" ? client.options : undefined; @@ -283,8 +308,7 @@ export class SimulationEnvironment { keyManagerPort: KEY_MANAGER_BASE_PORT + this.nodePairCount + 1, config: this.forkConfig, address: "127.0.0.1", - remoteKeys: options?.remoteKeys ?? [], - localKeys: options?.localKeys ?? [], + keys: options?.keys ?? {type: "no-keys"}, genesisTime: this.options.genesisTime, engineUrl: options?.engineUrl ?? `http://127.0.0.1:${EL_ENGINE_BASE_PORT + this.nodePairCount + 1}`, engineMock: options?.engineMock ?? false, diff --git a/packages/cli/test/utils/simulation/SimulationTracker.ts b/packages/cli/test/utils/simulation/SimulationTracker.ts index 84fb4a5b4e55..ec82e6f86a08 100644 --- a/packages/cli/test/utils/simulation/SimulationTracker.ts +++ b/packages/cli/test/utils/simulation/SimulationTracker.ts @@ -135,12 +135,28 @@ export class SimulationTracker { this.initEventStreamForNode(node); } this.reporter.bootstrap(); + + // Start clock loop on current slot or genesis + this.clockLoop(Math.max(this.clock.currentSlot, 0)).catch((e) => { + console.error("error on clockLoop", e); + }); } async stop(): Promise { // Do nothing; } + async clockLoop(slot: number): Promise { + while (!this.signal.aborted) { + // Wait for 2/3 of the slot to consider it missed + await this.clock.waitForStartOfSlot(slot + 2 / 3).catch((e) => { + console.error("error on waitForStartOfSlot", e); + }); + this.reporter.progress(slot); + slot++; + } + } + getErrorCount(): number { return this.errors.length; } diff --git a/packages/cli/test/utils/simulation/TableRenderer.ts b/packages/cli/test/utils/simulation/TableRenderer.ts index da7b4e677c42..1418853585cc 100644 --- a/packages/cli/test/utils/simulation/TableRenderer.ts +++ b/packages/cli/test/utils/simulation/TableRenderer.ts @@ -1,3 +1,5 @@ +import {strFixedSize} from "./utils/index.js"; + export class TableRenderer { private columnsSizes: Record; private columns: Columns[]; @@ -20,7 +22,7 @@ export class TableRenderer { addEmptyRow(text: string): void { this.printHSeparator(true); this.printVSeparator("start"); - process.stdout.write(text.padEnd(this.totalWidth - 4)); + process.stdout.write(strFixedSize(text, this.totalWidth - 4)); this.printVSeparator("end"); this.printHSeparator(true); } @@ -29,7 +31,7 @@ export class TableRenderer { this.printHSeparator(true); this.printVSeparator("start"); for (const [index, column] of this.columns.entries()) { - process.stdout.write(column.padEnd(this.columnsSizes[column])); + process.stdout.write(strFixedSize(column, this.columnsSizes[column])); if (index === this.columns.length - 1) { this.printVSeparator("end"); } else { diff --git a/packages/cli/test/utils/simulation/TableReporter.ts b/packages/cli/test/utils/simulation/TableReporter.ts index 63a945cf58b8..5827c1a8c436 100644 --- a/packages/cli/test/utils/simulation/TableReporter.ts +++ b/packages/cli/test/utils/simulation/TableReporter.ts @@ -6,6 +6,8 @@ import {TableRenderer} from "./TableRenderer.js"; import {arrayGroupBy, avg} from "./utils/index.js"; export class TableReporter extends SimulationReporter { + private lastPrintedSlot = -1; + private table = new TableRenderer({ fork: 10, eph: 5, @@ -23,73 +25,79 @@ export class TableReporter extends SimulationReporter } progress(slot: Slot): void { - { - const {clock, forkConfig, nodes, stores, errors} = this.options; - - const epoch = clock.getEpochForSlot(slot); - const forkName = forkConfig.getForkName(slot); - const epochStr = `${epoch}/${clock.getSlotIndexInEpoch(slot)}`; - - if (clock.isFirstSlotOfEpoch(slot)) { - // We are printing this info for last epoch - if (epoch - 1 < forkConfig.ALTAIR_FORK_EPOCH) { - this.table.addEmptyRow("Att Participation: N/A - SC Participation: N/A"); - } else { - // attestationParticipation is calculated at first slot of an epoch - const participation = nodes.map((node) => stores["attestationParticipation"][node.cl.id][slot] ?? 0); - const head = avg(participation.map((p) => p.head)).toFixed(2); - const source = avg(participation.map((p) => p.source)).toFixed(2); - const target = avg(participation.map((p) => p.target)).toFixed(2); - - // As it's printed on the first slot of epoch we need to get the previous epoch - const startSlot = clock.getFirstSlotOfEpoch(epoch - 1); - const endSlot = clock.getLastSlotOfEpoch(epoch - 1); - const nodesSyncParticipationAvg: number[] = []; - for (const node of nodes) { - const syncCommitteeParticipation: number[] = []; - for (let slot = startSlot; slot <= endSlot; slot++) { - syncCommitteeParticipation.push(stores["syncCommitteeParticipation"][node.cl.id][slot]); - } - nodesSyncParticipationAvg.push(avg(syncCommitteeParticipation)); + // Print slots once, may be called twice for missed block timer + if (slot <= this.lastPrintedSlot) { + return; + } else { + this.lastPrintedSlot = slot; + } + + const {clock, forkConfig, nodes, stores, errors} = this.options; + + const epoch = clock.getEpochForSlot(slot); + const forkName = forkConfig.getForkName(slot); + const epochStr = `${epoch}/${clock.getSlotIndexInEpoch(slot)}`; + + if (clock.isFirstSlotOfEpoch(slot)) { + // We are printing this info for last epoch + if (epoch - 1 < forkConfig.ALTAIR_FORK_EPOCH) { + this.table.addEmptyRow("Att Participation: N/A - SC Participation: N/A"); + } else { + // attestationParticipation is calculated at first slot of an epoch + const participation = nodes.map((node) => stores["attestationParticipation"][node.cl.id][slot] ?? 0); + const head = avg(participation.map((p) => p.head)).toFixed(2); + const source = avg(participation.map((p) => p.source)).toFixed(2); + const target = avg(participation.map((p) => p.target)).toFixed(2); + + // As it's printed on the first slot of epoch we need to get the previous epoch + const startSlot = clock.getFirstSlotOfEpoch(epoch - 1); + const endSlot = clock.getLastSlotOfEpoch(epoch - 1); + const nodesSyncParticipationAvg: number[] = []; + for (const node of nodes) { + const syncCommitteeParticipation: number[] = []; + for (let slot = startSlot; slot <= endSlot; slot++) { + syncCommitteeParticipation.push(stores["syncCommitteeParticipation"][node.cl.id][slot]); } + nodesSyncParticipationAvg.push(avg(syncCommitteeParticipation)); + } - const syncParticipation = avg(nodesSyncParticipationAvg).toFixed(2); + const syncParticipation = avg(nodesSyncParticipationAvg).toFixed(2); - this.table.addEmptyRow( - `Att Participation: H: ${head}, S: ${source}, T: ${target} - SC Participation: ${syncParticipation}` - ); - } + this.table.addEmptyRow( + `Att Participation: H: ${head}, S: ${source}, T: ${target} - SC Participation: ${syncParticipation}` + ); } + } - const finalizedSlots = nodes.map((node) => stores["finalized"][node.cl.id][slot] ?? "-"); - const finalizedSlotsUnique = new Set(finalizedSlots); + const finalizedSlots = nodes.map((node) => stores["finalized"][node.cl.id][slot] ?? "-"); + const finalizedSlotsUnique = new Set(finalizedSlots); - const inclusionDelay = nodes.map((node) => stores["inclusionDelay"][node.cl.id][slot] ?? "-"); - const inclusionDelayUnique = new Set(inclusionDelay); + const inclusionDelay = nodes.map((node) => stores["inclusionDelay"][node.cl.id][slot] ?? "-"); + const inclusionDelayUnique = new Set(inclusionDelay); - const attestationCount = nodes.map((node) => stores["attestationsCount"][node.cl.id][slot] ?? "-"); - const attestationCountUnique = new Set(attestationCount); + const attestationCount = nodes.map((node) => stores["attestationsCount"][node.cl.id][slot] ?? "-"); + const attestationCountUnique = new Set(attestationCount); - const head = nodes.map((node) => stores["head"][node.cl.id][slot] ?? "-"); - const headUnique = new Set(head); + const heads = nodes.map((node) => stores["head"][node.cl.id][slot]); + const head0 = heads.length > 0 ? heads[0] : null; + const nodesHaveSameHead = heads.every((head) => head?.blockRoot !== head0?.blockRoot); - const peerCount = nodes.map((node) => stores["connectedPeerCount"][node.cl.id][slot] ?? "-"); - const peerCountUnique = new Set(head); + const peerCount = nodes.map((node) => stores["connectedPeerCount"][node.cl.id][slot] ?? "-"); + const peerCountUnique = new Set(peerCount); - const errorCount = errors.filter((e) => e.slot === slot).length; + const errorCount = errors.filter((e) => e.slot === slot).length; - this.table.addRow({ - fork: forkName, - eph: epochStr, - slot: slot, - head: headUnique.size === 1 ? `${head[0].slice(0, 6)}..` : "different", - finzed: finalizedSlotsUnique.size === 1 ? finalizedSlots[0] : finalizedSlots.join(","), - peers: peerCountUnique.size === 1 ? peerCount[0] : peerCount.join(","), - attCount: attestationCountUnique.size === 1 ? attestationCount[0] : "---", - incDelay: inclusionDelayUnique.size === 1 ? inclusionDelay[0].toFixed(2) : "---", - errors: errorCount, - }); - } + this.table.addRow({ + fork: forkName, + eph: epochStr, + slot: head0 ? head0.slot : "-", + head: head0 ? (nodesHaveSameHead ? `${head0?.blockRoot.slice(0, 6)}..` : "different") : "-", + finzed: finalizedSlotsUnique.size === 1 ? finalizedSlots[0] : finalizedSlots.join(","), + peers: peerCountUnique.size === 1 ? peerCount[0] : peerCount.join(","), + attCount: attestationCountUnique.size === 1 ? attestationCount[0] : "---", + incDelay: inclusionDelayUnique.size === 1 ? inclusionDelay[0].toFixed(2) : "---", + errors: errorCount, + }); } summary(): void { diff --git a/packages/cli/test/utils/simulation/assertions/defaults/headAssertion.ts b/packages/cli/test/utils/simulation/assertions/defaults/headAssertion.ts index 72f1b465b39f..f8aef8ac8bb3 100644 --- a/packages/cli/test/utils/simulation/assertions/defaults/headAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/defaults/headAssertion.ts @@ -1,25 +1,35 @@ +import {RootHex, Slot} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {SimulationAssertion} from "../../interfaces.js"; import {everySlotMatcher} from "../matchers.js"; -export const headAssertion: SimulationAssertion<"head", string> = { +export interface HeadSummary { + blockRoot: RootHex; + slot: Slot; +} + +export const headAssertion: SimulationAssertion<"head", HeadSummary> = { id: "head", async capture({node}) { const head = await node.cl.api.beacon.getBlockHeader("head"); - return toHexString(head.data.root); + + return { + blockRoot: toHexString(head.data.root), + slot: head.data.header.message.slot, + }; }, match: everySlotMatcher, async assert({nodes, store, slot}) { const errors: string[] = []; - const headOnFirstNode = store[nodes[0].cl.id][slot]; + const headRootNode0 = store[nodes[0].cl.id][slot].blockRoot; for (let i = 1; i < nodes.length; i++) { - const headOnNNode = store[nodes[i].cl.id][slot]; + const headRootNodeN = store[nodes[i].cl.id][slot].blockRoot; - if (headOnFirstNode !== headOnNNode) { - errors.push(`node have different heads. ${JSON.stringify({slot, headOnFirstNode, headOnNNode})}`); + if (headRootNode0 !== headRootNodeN) { + errors.push(`node have different heads. ${JSON.stringify({slot, headRootNode0, headRootNodeN})}`); } } diff --git a/packages/cli/test/utils/simulation/assertions/defaults/missedBlocksAssertion.ts b/packages/cli/test/utils/simulation/assertions/defaults/missedBlocksAssertion.ts index fa11be9f9871..71e77e7e5d0b 100644 --- a/packages/cli/test/utils/simulation/assertions/defaults/missedBlocksAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/defaults/missedBlocksAssertion.ts @@ -16,7 +16,7 @@ export const missedBlocksAssertion: SimulationAssertion<"missedBlocks", number[] const missedSlots: number[] = []; for (let slot = startSlot; slot < endSlot; slot++) { - if (!dependantStores["head"][node.cl.id][slot]) { + if (!dependantStores["head"][node.cl.id][slot].slot) { missedSlots.push(slot); } } diff --git a/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts b/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts index a0ed64c7570f..1356b94d50c5 100644 --- a/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/nodeAssertion.ts @@ -1,5 +1,6 @@ import {routes} from "@lodestar/api/beacon"; -import {SimulationAssertion} from "../interfaces.js"; +import type {SecretKey} from "@chainsafe/bls/types"; +import {CLClientKeys, SimulationAssertion} from "../interfaces.js"; import {arrayEquals} from "../utils/index.js"; import {neverMatcher} from "./matchers.js"; @@ -17,17 +18,14 @@ export const nodeAssertion: SimulationAssertion<"node", string> = { errors.push(`node health is neither READY or SYNCING. ${JSON.stringify({id: node.cl.id})}`); } - const keyManagerKeys = (await node.cl.keyManager.listKeys()).data.map((k) => k.validatingPubkey).sort(); - const existingKeys = [ - ...node.cl.remoteKeys.map((k) => k.toPublicKey().toHex()), - ...node.cl.localKeys.map((k) => k.toPublicKey().toHex()), - ].sort(); + const keyManagerKeys = (await node.cl.keyManager.listKeys()).data.map((k) => k.validatingPubkey); + const expectedPubkeys = getAllKeys(node.cl.keys).map((k) => k.toPublicKey().toHex()); - if (!arrayEquals(keyManagerKeys, existingKeys)) { + if (!arrayEquals(keyManagerKeys.sort(), expectedPubkeys.sort())) { errors.push( `Validator should have correct number of keys loaded. ${JSON.stringify({ id: node.cl.id, - existingKeys, + expectedPubkeys, keyManagerKeys, })}` ); @@ -37,3 +35,14 @@ export const nodeAssertion: SimulationAssertion<"node", string> = { return errors; }, }; + +function getAllKeys(keys: CLClientKeys): SecretKey[] { + switch (keys.type) { + case "local": + return keys.secretKeys; + case "remote": + return keys.secretKeys; + case "no-keys": + return []; + } +} diff --git a/packages/cli/test/utils/simulation/cl_clients/lodestar.ts b/packages/cli/test/utils/simulation/cl_clients/lodestar.ts index d8b9c46d1bdf..0de886db33a9 100644 --- a/packages/cli/test/utils/simulation/cl_clients/lodestar.ts +++ b/packages/cli/test/utils/simulation/cl_clients/lodestar.ts @@ -26,8 +26,7 @@ export const generateLodestarBeaconNode: CLClientGenerator = id, config, genesisStateFilePath, - remoteKeys, - localKeys, + keys, keyManagerPort, genesisTime, engineUrl, @@ -78,7 +77,7 @@ export const generateLodestarBeaconNode: CLClientGenerator = } const validatorClientsJobs: JobOptions[] = []; - if (opts.localKeys.length > 0 || opts.remoteKeys.length > 0) { + if (keys.type !== "no-keys") { validatorClientsJobs.push( generateLodestarValidatorJobs( { @@ -94,6 +93,7 @@ export const generateLodestarBeaconNode: CLClientGenerator = const job = runner.create(id, [ { + id, bootstrap: async () => { await mkdir(dataDir, {recursive: true}); await writeFile(rcConfigPath, JSON.stringify(rcConfig, null, 2)); @@ -104,7 +104,7 @@ export const generateLodestarBeaconNode: CLClientGenerator = command: LODESTAR_BINARY_PATH, args: ["beacon", "--rcConfig", rcConfigPath, "--paramsFile", paramsPath], env: { - DEBUG: "*,-winston:*", + DEBUG: process.env.DISABLE_DEBUG_LOGS ? "" : "*,-winston:*", }, }, logs: { @@ -113,9 +113,9 @@ export const generateLodestarBeaconNode: CLClientGenerator = health: async () => { try { await got.get(`http://${address}:${restPort}/eth/v1/node/health`); - return true; - } catch { - return false; + return {ok: true}; + } catch (err) { + return {ok: false, reason: (err as Error).message, checkId: "eth/v1/node/health query"}; } }, children: validatorClientsJobs, @@ -126,8 +126,7 @@ export const generateLodestarBeaconNode: CLClientGenerator = id, client: CLClient.Lodestar, url: `http://${address}:${restPort}`, - localKeys, - remoteKeys, + keys, api: getClient({baseUrl: `http://${address}:${restPort}`}, {config}), keyManager: keyManagerGetClient({baseUrl: `http://${address}:${keyManagerPort}`}, {config}), }; @@ -143,7 +142,11 @@ export const generateLodestarValidatorJobs = ( throw new Error(`Runner "${runner.type}" not yet supported.`); } - const {dataDir: rootDir, id, address, keyManagerPort, localKeys, restPort, config, genesisTime} = opts; + const {dataDir: rootDir, id, address, keyManagerPort, restPort, keys, config, genesisTime} = opts; + + if (keys.type === "no-keys") { + throw Error("Attempting to run a vc with keys.type == 'no-keys'"); + } const rcConfig = ({ network: "dev", @@ -163,6 +166,7 @@ export const generateLodestarValidatorJobs = ( } as unknown) as IValidatorCliArgs & IGlobalArgs; return { + id, bootstrap: async () => { await mkdir(rootDir); await mkdir(`${rootDir}/keystores`); @@ -170,19 +174,21 @@ export const generateLodestarValidatorJobs = ( await writeFile(join(rootDir, "rc_config.json"), JSON.stringify(rcConfig, null, 2)); await writeFile(join(rootDir, "params.json"), JSON.stringify(chainConfigToJson(config), null, 2)); - for (const key of localKeys) { - const keystore = await Keystore.create("password", key.toBytes(), key.toPublicKey().toBytes(), ""); - await writeFile( - join(rootDir, "keystores", `${key.toPublicKey().toHex()}.json`), - JSON.stringify(keystore.toObject(), null, 2) - ); + if (keys.type === "local") { + for (const key of keys.secretKeys) { + const keystore = await Keystore.create("password", key.toBytes(), key.toPublicKey().toBytes(), ""); + await writeFile( + join(rootDir, "keystores", `${key.toPublicKey().toHex()}.json`), + JSON.stringify(keystore.toObject(), null, 2) + ); + } } }, cli: { command: LODESTAR_BINARY_PATH, args: ["validator", "--rcConfig", join(rootDir, "rc_config.json"), "--paramsFile", join(rootDir, "params.json")], env: { - DEBUG: "*,-winston:*", + DEBUG: process.env.DISABLE_DEBUG_LOGS ? "" : "*,-winston:*", }, }, logs: { @@ -191,9 +197,9 @@ export const generateLodestarValidatorJobs = ( health: async () => { try { await got.get(`http://${address}:${keyManagerPort}/eth/v1/keystores`); - return true; + return {ok: true}; } catch (err) { - return false; + return {ok: false, reason: (err as Error).message, checkId: "eth/v1/keystores query"}; } }, }; diff --git a/packages/cli/test/utils/simulation/constants.ts b/packages/cli/test/utils/simulation/constants.ts index eb8a92640b0e..4d528bdc1eca 100644 --- a/packages/cli/test/utils/simulation/constants.ts +++ b/packages/cli/test/utils/simulation/constants.ts @@ -20,3 +20,5 @@ export const ETH_TTD_INCREMENT = 2; export const SIM_ENV_CHAIN_ID = 1234; export const SIM_ENV_NETWORK_ID = 1234; export const LODESTAR_BINARY_PATH = `${__dirname}/../../../bin/lodestar.js`; +export const MOCK_ETH1_GENESIS_HASH = "0xfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb"; +export const SHARED_JWT_SECRET = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; diff --git a/packages/cli/test/utils/simulation/el_clients/geth.ts b/packages/cli/test/utils/simulation/el_clients/geth.ts index 9ac1989a05cb..9a66c7dd90e0 100644 --- a/packages/cli/test/utils/simulation/el_clients/geth.ts +++ b/packages/cli/test/utils/simulation/el_clients/geth.ts @@ -67,6 +67,7 @@ export const generateGethNode: ELClientGenerator = ( const jwtSecretGethPath = join(gethDataDir, "jwtsecret"); const initJobOptions: JobOptions = { + id: `${id}-init`, bootstrap: async () => { await mkdir(dataDir, {recursive: true}); await writeFile(genesisPath, JSON.stringify(getGethGenesisBlock(mode, {ttd, cliqueSealingPeriod}))); @@ -82,6 +83,7 @@ export const generateGethNode: ELClientGenerator = ( }; const importJobOptions: JobOptions = { + id: `${id}-import`, bootstrap: async () => { await writeFile(skPath, SECRET_KEY); await writeFile(passwordPath, PASSWORD); @@ -108,6 +110,7 @@ export const generateGethNode: ELClientGenerator = ( }; const startJobOptions: JobOptions = { + id, cli: { command: binaryPath, args: [ @@ -151,12 +154,12 @@ export const generateGethNode: ELClientGenerator = ( logs: { stdoutFilePath: logFilePath, }, - health: async (): Promise => { + health: async () => { try { await got.post(ethRpcUrl, {json: {jsonrpc: "2.0", method: "net_version", params: [], id: 67}}); - return true; - } catch (e) { - return false; + return {ok: true}; + } catch (err) { + return {ok: false, reason: (err as Error).message, checkId: "JSON RPC query net_version"}; } }, }; diff --git a/packages/cli/test/utils/simulation/el_clients/nethermind.ts b/packages/cli/test/utils/simulation/el_clients/nethermind.ts index ccf7058f5d9d..b3ab425205c8 100644 --- a/packages/cli/test/utils/simulation/el_clients/nethermind.ts +++ b/packages/cli/test/utils/simulation/el_clients/nethermind.ts @@ -46,6 +46,7 @@ export const generateNethermindNode: ELClientGenerator = ( const jwtSecretContainerPath = join(containerDataDir, "jwtsecret"); const startJobOptions: JobOptions = { + id, bootstrap: async () => { await mkdir(dataDir, {recursive: true}); await writeFile(chainSpecPath, JSON.stringify(getNethermindChainSpec(mode, {ttd, cliqueSealingPeriod}))); @@ -99,12 +100,12 @@ export const generateNethermindNode: ELClientGenerator = ( logs: { stdoutFilePath: logFilePath, }, - health: async (): Promise => { + health: async () => { try { await got.post(ethRpcUrl, {json: {jsonrpc: "2.0", method: "net_version", params: [], id: 67}}); - return true; - } catch (e) { - return false; + return {ok: true}; + } catch (err) { + return {ok: false, reason: (err as Error).message, checkId: "JSON RPC query net_version"}; } }, }; diff --git a/packages/cli/test/utils/simulation/interfaces.ts b/packages/cli/test/utils/simulation/interfaces.ts index f98dc15316d1..27975c93b8e7 100644 --- a/packages/cli/test/utils/simulation/interfaces.ts +++ b/packages/cli/test/utils/simulation/interfaces.ts @@ -60,6 +60,11 @@ export interface NodePairOptions { id: string; dataDir: string; @@ -70,8 +75,7 @@ export interface CLClientGeneratorOptions { port: number; keyManagerPort: number; config: IChainForkConfig; - localKeys: SecretKey[]; - remoteKeys: SecretKey[]; + keys: CLClientKeys; genesisTime: number; engineUrl: string; engineMock: boolean; @@ -104,8 +108,7 @@ export interface CLNode { readonly url: string; readonly api: Api; readonly keyManager: KeyManagerApi; - readonly localKeys: SecretKey[]; - readonly remoteKeys: SecretKey[]; + readonly keys: CLClientKeys; } export interface ELNode { @@ -143,7 +146,11 @@ export type ELClientGenerator = ( runner: Runner | Runner ) => {job: Job; node: ELNode}; +export type HealthStatus = {ok: true} | {ok: false; reason: string; checkId: string}; + export interface JobOptions { + readonly id: string; + readonly cli: { readonly command: string; readonly args: string[]; @@ -159,7 +166,7 @@ export interface JobOptions { // Will be called frequently to check the health of job startup // If not present then wait for the job to exit - health?(): Promise; + health?(): Promise; // Called once before the `job.start` is called bootstrap?(): Promise; diff --git a/packages/cli/test/utils/simulation/runner/DockerRunner.ts b/packages/cli/test/utils/simulation/runner/DockerRunner.ts index 2e936ea53360..f9b695081b14 100644 --- a/packages/cli/test/utils/simulation/runner/DockerRunner.ts +++ b/packages/cli/test/utils/simulation/runner/DockerRunner.ts @@ -59,6 +59,7 @@ const convertJobOptionsToDocker = ( const connectContainerToNetwork = async (container: string, ip: string, logFilePath: string): Promise => { await startChildProcess({ + id: `connect ${container} to network ${dockerNetworkName}`, cli: { command: "docker", args: ["network", "connect", dockerNetworkName, container, "--ip", ip], @@ -83,6 +84,7 @@ export class DockerRunner implements Runner { async start(): Promise { try { await startChildProcess({ + id: `create docker network '${dockerNetworkName}'`, cli: { command: "docker", args: ["network", "create", "--subnet", `${dockerNetworkIpRange}.0/24`, dockerNetworkName], @@ -102,6 +104,7 @@ export class DockerRunner implements Runner { for (let i = 0; i < 5; i++) { try { await startChildProcess({ + id: `docker network rm '${dockerNetworkName}'`, cli: { command: "docker", args: ["network", "rm", dockerNetworkName], @@ -135,7 +138,7 @@ export class DockerRunner implements Runner { const dockerJobOptions = convertJobOptionsToDocker(jobs, id, {image, dataVolumePath, exposePorts, dockerNetworkIp}); const stop = async (): Promise => { - console.log(`Stopping "${id}"...`); + console.log(`DockerRunner stopping '${id}'...`); this.emitter.emit("stopping"); for (const {jobOptions, childProcess} of childProcesses) { if (jobOptions.teardown) { @@ -144,7 +147,7 @@ export class DockerRunner implements Runner { await stopChildProcess(childProcess); } - console.log(`Stopped "${id}"...`); + console.log(`DockerRunner stopped '${id}'`); this.emitter.emit("stopped"); }; @@ -152,15 +155,14 @@ export class DockerRunner implements Runner { new Promise((resolve, reject) => { void (async () => { try { - // eslint-disable-next-line no-console console.log(`Starting "${id}"...`); this.emitter.emit("starting"); childProcesses.push(...(await startJobs(dockerJobOptions))); - // eslint-disable-next-line no-console console.log(`Started "${id}"...`); this.emitter.emit("started"); await connectContainerToNetwork(id, dockerNetworkIp, this.logFilePath); + console.log(`DockerRunner connected container to network '${id}'`); resolve(); } catch (err) { reject(err); diff --git a/packages/cli/test/utils/simulation/utils/child_process.ts b/packages/cli/test/utils/simulation/utils/child_process.ts index aef1f4004132..e95a3d9fb814 100644 --- a/packages/cli/test/utils/simulation/utils/child_process.ts +++ b/packages/cli/test/utils/simulation/utils/child_process.ts @@ -1,9 +1,11 @@ +/* eslint-disable no-console */ import {ChildProcess, spawn} from "node:child_process"; import {createWriteStream, mkdirSync} from "node:fs"; import {dirname} from "node:path"; import {ChildProcessWithJobOptions, JobOptions} from "../interfaces.js"; const childProcessHealthCheckInterval = 1000; +const logHealthChecksAfterMs = 2000; export const stopChildProcess = async ( childProcess: ChildProcess, @@ -36,13 +38,30 @@ export const startChildProcess = async (jobOptions: JobOptions): Promise { - if (jobOptions.health && (await jobOptions.health())) { - clearInterval(intervalId); - childProcess.removeAllListeners("exit"); - resolve(childProcess); - } + const health = jobOptions.health; + + // If there is a health check, wait for it to pass + if (health) { + const startHealthCheckMs = Date.now(); + const intervalId = setInterval(() => { + health() + .then((isHealthy) => { + if (isHealthy.ok) { + clearInterval(intervalId); + childProcess.removeAllListeners("exit"); + resolve(childProcess); + } else { + const timeSinceHealthCheckStart = Date.now() - startHealthCheckMs; + if (timeSinceHealthCheckStart > logHealthChecksAfterMs) { + // eslint-disable-next-line no-console + console.log(`Health check unsuccessful '${jobOptions.id}' after ${timeSinceHealthCheckStart} ms`); + } + } + }) + .catch((e) => { + // eslint-disable-next-line no-console + console.error("error on health check, health functions must never throw", e); + }); }, childProcessHealthCheckInterval); childProcess.once("exit", (code: number) => { @@ -52,21 +71,19 @@ export const startChildProcess = async (jobOptions: JobOptions): Promise { + stdoutFileStream.close(); + if (code > 0) { + reject( + new Error(`$process exited with code ${code}. ${jobOptions.cli.command} ${jobOptions.cli.args.join(" ")}`) + ); + } else { + resolve(childProcess); + } + }); } - - // If there is no health check, resolve/reject on completion - childProcess.once("exit", (code: number) => { - stdoutFileStream.close(); - if (code > 0) { - reject( - new Error(`$process exited with code ${code}. ${jobOptions.cli.command} ${jobOptions.cli.args.join(" ")}`) - ); - } else { - resolve(childProcess); - } - }); })(); }); }; @@ -75,9 +92,13 @@ export const startJobs = async (jobs: JobOptions[]): Promise( (acc[predicate(value, index, array)] ||= []).push(value); return acc; }, {} as {[key: string]: T[]}); + +export function strFixedSize(str: string, width: number): string { + return str.padEnd(width).slice(0, width); +}