From cb6b3e1302ac5d9d7bf922d18a4ff894406a8214 Mon Sep 17 00:00:00 2001 From: Hany Date: Fri, 16 May 2025 16:59:42 +0300 Subject: [PATCH 1/4] feat: add retrievalMetrics module to track per-round coverage and failures --- lib/metrics.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 lib/metrics.js diff --git a/lib/metrics.js b/lib/metrics.js new file mode 100644 index 0000000..cacdd37 --- /dev/null +++ b/lib/metrics.js @@ -0,0 +1,67 @@ +/** + * Tracks per-round retrieval metrics for SPARK: + * - Total number of retrieval attempts + * - Number of failed retrievals + * - Number of unique (PayloadCID, SP) pairs attempted + */ + +class RetrievalMetrics { + constructor() { + this.roundIndex = 0 + this.retrievalsTotal = 0 + this.retrievalsFailed = 0 + this.uniquePairs = new Set() + } + + /** + * Called at the start of a new SPARK round. + */ + reset() { + this.retrievalsTotal = 0 + this.retrievalsFailed = 0 + this.uniquePairs.clear() + this.roundIndex++ + } + + /** + * Register a successful or attempted retrieval. + * @param {string} payloadCID + * @param {string|number} storageProvider + */ + recordRetrieval(payloadCID, storageProvider) { + this.retrievalsTotal += 1 + const key = `${payloadCID}:${storageProvider}` + this.uniquePairs.add(key) + } + + /** + * Must be called separately on error). + */ + recordFailure() { + this.retrievalsFailed += 1 + } + + /** + * Log metrics for the current round. + */ + report() { + console.log(`[METRICS] Round #${this.roundIndex}`) + console.log(` Retrievals attempted: ${this.retrievalsTotal}`) + console.log(` Retrievals failed: ${this.retrievalsFailed}`) + console.log(` Unique (PayloadCID, SP) pairs: ${this.uniquePairs.size}`) + } + + /** + * Get snapshot of current state. + */ + getSnapshot() { + return { + round: this.roundIndex, + total: this.retrievalsTotal, + failed: this.retrievalsFailed, + uniquePairCount: this.uniquePairs.size + } + } +} + +export const retrievalMetrics = new RetrievalMetrics() From 64700092bd2cd9bfed26aa7f2495ec81f06c893d Mon Sep 17 00:00:00 2001 From: Hany Date: Fri, 16 May 2025 18:36:50 +0300 Subject: [PATCH 2/4] feat: track and report per-round retrieval metrics (total, failures, unique pairs) --- lib/spark.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/spark.js b/lib/spark.js index 86e725d..e79ecc1 100644 --- a/lib/spark.js +++ b/lib/spark.js @@ -13,6 +13,7 @@ import { assertOkResponse } from './http-assertions.js' import { getIndexProviderPeerId as defaultGetIndexProvider } from './miner-info.js' import { multiaddrToHttpUrl } from './multiaddr.js' import { Tasker } from './tasker.js' +import { retrievalMetrics } from './metrics.js' import { CarBlockIterator, @@ -255,16 +256,24 @@ export default class Spark { } const stats = newStats() + retrievalMetrics.recordRetrieval(retrieval.cid, retrieval.minerId) - await this.executeRetrievalCheck(retrieval, stats) - - const measurementId = await this.submitMeasurement(retrieval, { ...stats }) - Zinnia.jobCompleted() - return measurementId - } + //Track failed attempts + try { + await this.executeRetrievalCheck(retrieval, stats) + const measurementId = await this.submitMeasurement(retrieval, { ...stats }) + Zinnia.jobCompleted() + return measurementId + } catch (err) { + retrievalMetrics.recordFailure() + throw err + } + } async run() { while (true) { + retrievalMetrics.reset() + const started = Date.now() try { await this.nextRetrieval() @@ -288,6 +297,7 @@ export default class Spark { await sleep(delay) console.log() // add an empty line to visually delimit logs from different tasks } + retrievalMetrics.report() } } From f96caff55d59d66d3c7ec2a2e8797fe7b10afaf2 Mon Sep 17 00:00:00 2001 From: Hany Date: Fri, 16 May 2025 18:58:50 +0300 Subject: [PATCH 3/4] test: cover retrievalMetrics logic for counts, deduplication, and rounds --- test/retrieval-metrics.test.js | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test/retrieval-metrics.test.js diff --git a/test/retrieval-metrics.test.js b/test/retrieval-metrics.test.js new file mode 100644 index 0000000..72e3115 --- /dev/null +++ b/test/retrieval-metrics.test.js @@ -0,0 +1,47 @@ +import { test } from 'zinnia:test' +import { assertEquals } from 'zinnia:assert' +import { retrievalMetrics } from '../lib/metrics.js' + +test('retrievalMetrics resets correctly', () => { + retrievalMetrics.reset() + const snap = retrievalMetrics.getSnapshot() + + assertEquals(snap.total, 0) + assertEquals(snap.failed, 0) + assertEquals(snap.uniquePairCount, 0) +}) + +test('retrievalMetrics counts retrievals and failures', () => { + retrievalMetrics.reset() + + retrievalMetrics.recordRetrieval('cid1', 'sp1') + retrievalMetrics.recordRetrieval('cid2', 'sp2') + retrievalMetrics.recordFailure() + + const snap = retrievalMetrics.getSnapshot() + + assertEquals(snap.total, 2) + assertEquals(snap.failed, 1) + assertEquals(snap.uniquePairCount, 2) +}) + +test('retrievalMetrics deduplicates (PayloadCID, SP) pairs', () => { + retrievalMetrics.reset() + + retrievalMetrics.recordRetrieval('cid1', 'sp1') + retrievalMetrics.recordRetrieval('cid1', 'sp1') + retrievalMetrics.recordRetrieval('cid2', 'sp2') + + const snap = retrievalMetrics.getSnapshot() + + assertEquals(snap.total, 3) + assertEquals(snap.uniquePairCount, 2) +}) + +test('retrievalMetrics roundIndex increments on reset', () => { + const before = retrievalMetrics.roundIndex + retrievalMetrics.reset() + const after = retrievalMetrics.roundIndex + + assertEquals(after, before + 1) +}) From b81020991e2f637998a1bf378e0d7c827acd91bb Mon Sep 17 00:00:00 2001 From: Hany Date: Fri, 16 May 2025 19:03:04 +0300 Subject: [PATCH 4/4] style: apply Prettier and ESLint fixes --- lib/metrics.js | 24 +++++++++--------------- lib/spark.js | 8 +++++--- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/lib/metrics.js b/lib/metrics.js index cacdd37..65439ab 100644 --- a/lib/metrics.js +++ b/lib/metrics.js @@ -1,5 +1,6 @@ /** * Tracks per-round retrieval metrics for SPARK: + * * - Total number of retrieval attempts * - Number of failed retrievals * - Number of unique (PayloadCID, SP) pairs attempted @@ -13,9 +14,7 @@ class RetrievalMetrics { this.uniquePairs = new Set() } - /** - * Called at the start of a new SPARK round. - */ + /** Called at the start of a new SPARK round. */ reset() { this.retrievalsTotal = 0 this.retrievalsFailed = 0 @@ -25,8 +24,9 @@ class RetrievalMetrics { /** * Register a successful or attempted retrieval. - * @param {string} payloadCID - * @param {string|number} storageProvider + * + * @param {string} payloadCID + * @param {string | number} storageProvider */ recordRetrieval(payloadCID, storageProvider) { this.retrievalsTotal += 1 @@ -34,16 +34,12 @@ class RetrievalMetrics { this.uniquePairs.add(key) } - /** - * Must be called separately on error). - */ + /** Must be called separately on error). */ recordFailure() { this.retrievalsFailed += 1 } - /** - * Log metrics for the current round. - */ + /** Log metrics for the current round. */ report() { console.log(`[METRICS] Round #${this.roundIndex}`) console.log(` Retrievals attempted: ${this.retrievalsTotal}`) @@ -51,15 +47,13 @@ class RetrievalMetrics { console.log(` Unique (PayloadCID, SP) pairs: ${this.uniquePairs.size}`) } - /** - * Get snapshot of current state. - */ + /** Get snapshot of current state. */ getSnapshot() { return { round: this.roundIndex, total: this.retrievalsTotal, failed: this.retrievalsFailed, - uniquePairCount: this.uniquePairs.size + uniquePairCount: this.uniquePairs.size, } } } diff --git a/lib/spark.js b/lib/spark.js index e79ecc1..f5bb4d1 100644 --- a/lib/spark.js +++ b/lib/spark.js @@ -261,19 +261,21 @@ export default class Spark { //Track failed attempts try { await this.executeRetrievalCheck(retrieval, stats) - const measurementId = await this.submitMeasurement(retrieval, { ...stats }) + const measurementId = await this.submitMeasurement(retrieval, { + ...stats, + }) Zinnia.jobCompleted() return measurementId } catch (err) { retrievalMetrics.recordFailure() throw err - } } + } async run() { while (true) { retrievalMetrics.reset() - + const started = Date.now() try { await this.nextRetrieval()