From 042cf311da78094254b76c1b6933a140eeff5b6b Mon Sep 17 00:00:00 2001 From: esgama107 Date: Thu, 26 Mar 2026 18:00:14 -0700 Subject: [PATCH 1/3] Trigger workflow discovery From aeb53851f2facc58f55cb0dbb027e1d3fb32cffe Mon Sep 17 00:00:00 2001 From: "Erick Javier Sanchez Gama ((HE/HIM)) (from Dev Box)" Date: Wed, 25 Mar 2026 14:52:24 -0700 Subject: [PATCH 2/3] Fix regression detection: enable Method 2, add key mapping, restore summary columns Regression detection has been completely non-functional since Nov 2024 due to 4 compounding bugs. This commit fixes all of them: 1. regression.py Method 1 path bug (line 81): fixed file.json/file.json -> file.json 2. regression.py key mapping: added DB_TO_CONSUMER_KEY dict that translates both old-format (tput-up-quic) and new scenario-* format (scenario-upload-quic) DB test IDs to consumer keys (upload-quic) expected by secnetperf.ps1 3. quic.yml: regression-detection job now fetches regression.py from main via wget (instead of stale sqlite branch copy) and runs with --featureint 2 to enable the sliding window method 4. generate-summary.ps1: uncommented regression columns (Avg, Noise, Best Ever) in all summary tables and added matching column headers Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/generate-summary.ps1 | 43 +++++++++++----------- .github/workflows/quic.yml | 4 ++- pipeline/regression.py | 50 ++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/.github/workflows/generate-summary.ps1 b/.github/workflows/generate-summary.ps1 index 53805dab7..0d0f4acbf 100644 --- a/.github/workflows/generate-summary.ps1 +++ b/.github/workflows/generate-summary.ps1 @@ -51,11 +51,10 @@ function Write-ThroughputRow { else { $row += " $(($Results[$i] / 1000000).ToString('F2')) |" } } - # TODO: Regression detection is heinously broken. Let's reduce the noise. - # $row += " " + $Regression.CumulativeResult + " |" - # $row += " " + $Regression.Baseline + " |" - # if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" } - # else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" } + $row += " " + $Regression.CumulativeResult + " |" + $row += " " + $Regression.Baseline + " |" + if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" } + else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" } $Script:markdown += $row } @@ -79,11 +78,10 @@ function Write-HpsRow { else { $row += " $($Results[$i]) |" } } - # TODO: Regression detection is heinously broken. Let's reduce the noise. - # $row += " " + $Regression.CumulativeResult + " |" - # $row += " " + $Regression.Baseline + " |" - # if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" } - # else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" } + $row += " " + $Regression.CumulativeResult + " |" + $row += " " + $Regression.Baseline + " |" + if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" } + else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" } $Script:markdown += $row } @@ -108,11 +106,10 @@ function Write-RpsRow { $row += " $($Results[$i+$j]) |" } - # TODO: Regression detection is heinously broken. Let's reduce the noise. - # $row += " " + $Regression.CumulativeResult + " |" - # $row += " " + $Regression.Baseline + " |" - # if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" } - # else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" } + $row += " " + $Regression.CumulativeResult + " |" + $row += " " + $Regression.Baseline + " |" + if ($Regression.BestResultCommit -eq "N/A") { $row += "N/A |" } + else { $row += "[" + $Regression.BestResult + "](https://github.com/microsoft/msquic/commit/" + $Regression.BestResultCommit + ") |" } $Script:markdown += $row } @@ -131,8 +128,8 @@ $hasRegression = $false # Write the Upload table. $markdown = @" # Upload Throughput (Gbps) -| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | -| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | +| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | Avg | Noise | Best Ever | +| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | --- | ----- | --------- | "@ foreach ($file in $files) { Write-Host "Upload Tput: Processing $file..." @@ -180,8 +177,8 @@ foreach ($file in $files) { $markdown += @" `n # Download Throughput (Gbps) -| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | -| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | +| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | Avg | Noise | Best Ever | +| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | --- | ----- | --------- | "@ foreach ($file in $files) { Write-Host "Download Tput: Processing $file..." @@ -228,8 +225,8 @@ foreach ($file in $files) { $markdown += @" `n # Handshakes Per Second (HPS) -| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | -| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | +| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Result 1 | Result 2 | Result 3 | Avg | Noise | Best Ever | +| --------- | --- | -- | ------- | ---- | --- | -- | --------- | -------- | -------- | -------- | --- | ----- | --------- | "@ foreach ($file in $files) { Write-Host "HPS: Processing $file..." @@ -275,8 +272,8 @@ foreach ($file in $files) { $markdown += @" `n # Request Per Second (RPS) and Latency (µs) -| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Min | P50 | P90 | P99 | P99.9 | P99.99 | P99.999 | P99.9999 | RPS | -| --------- | --- | -- | ------- | ---- | --- | -- | --------- | --- | --- | --- | --- | ----- | ------ | ------- | -------- | --- | +| Pass/Fail | Env | OS | Version | Arch | TLS | IO | Transport | Min | P50 | P90 | P99 | P99.9 | P99.99 | P99.999 | P99.9999 | RPS | Avg | Noise | Best Ever | +| --------- | --- | -- | ------- | ---- | --- | -- | --------- | --- | --- | --- | --- | ----- | ------ | ------- | -------- | --- | --- | ----- | --------- | "@ foreach ($file in $files) { # TODO: Right now, we are not using a watermark based method for regression detection of latency percentile values because we don't know how to determine a "Best Ever" distribution. diff --git a/.github/workflows/quic.yml b/.github/workflows/quic.yml index e5395e570..f6df4d8ec 100644 --- a/.github/workflows/quic.yml +++ b/.github/workflows/quic.yml @@ -477,7 +477,9 @@ jobs: repository: microsoft/netperf ref: sqlite - run: ls - - run: python regression.py + - name: Fetch regression.py from main branch + run: wget https://raw.githubusercontent.com/microsoft/netperf/main/pipeline/regression.py -O regression.py + - run: python regression.py --featureint 2 - name: Git commit if: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.pr == '') || inputs.commit }} run: 'git config user.name "QUIC Dev[bot]" && git config user.email "quicdev@microsoft.com" && git add *.json && git commit -m "Update regression metrics" && git push' diff --git a/pipeline/regression.py b/pipeline/regression.py index 4c63c4b78..59e9c493c 100644 --- a/pipeline/regression.py +++ b/pipeline/regression.py @@ -5,6 +5,48 @@ import glob import os +# Mapping from DB test IDs (old-format and new scenario-* format) to consumer keys. +# The consumer (secnetperf.ps1 / secnetperf-helpers.psm1) builds keys as "$scenario-$transport", +# e.g. "upload-quic", "download-tcp". The DB uses different naming conventions. +DB_TO_CONSUMER_KEY = { + # Old format → consumer + "tput-up-quic": "upload-quic", + "tput-up-tcp": "upload-tcp", + "tput-down-quic": "download-quic", + "tput-down-tcp": "download-tcp", + "hps-conns-100-quic": "hps-quic", + "hps-conns-100-tcp": "hps-tcp", + "rps-up-512-down-4000-quic": "rps-quic", + "rps-up-512-down-4000-tcp": "rps-tcp", + "max-rps-up-512-down-4000-quic": "rps-quic", + "max-rps-up-512-down-4000-tcp": "rps-tcp", + # New scenario-* format → consumer + "scenario-upload-quic": "upload-quic", + "scenario-upload-tcp": "upload-tcp", + "scenario-download-quic": "download-quic", + "scenario-download-tcp": "download-tcp", + "scenario-hps-quic": "hps-quic", + "scenario-hps-tcp": "hps-tcp", + "scenario-rps-quic": "rps-quic", + "scenario-rps-tcp": "rps-tcp", + "scenario-rps-multi-quic": "rps-multi-quic", + "scenario-rps-multi-tcp": "rps-multi-tcp", + "scenario-latency-quic": "latency-quic", + "scenario-latency-tcp": "latency-tcp", +} + + +def remap_to_consumer_keys(data): + """Remap DB test IDs to consumer-format keys, merging env data when collisions occur.""" + result = {} + for db_key, env_data in data.items(): + consumer_key = DB_TO_CONSUMER_KEY.get(db_key, db_key) + if consumer_key not in result: + result[consumer_key] = {} + # Merge: newer data (scenario-*) overwrites older data for same env + result[consumer_key].update(env_data) + return result + # Create the parser parser = argparse.ArgumentParser(description="Process a feature integer.") @@ -78,10 +120,10 @@ def singular_datapoint_method(): watermark_regression_file = {} for json_file_path in glob.glob('*.json'): - if not os.path.exists(f"{json_file_path}/{json_file_path}"): + if not os.path.exists(json_file_path): continue - with open(f"{json_file_path}/{json_file_path}", 'r') as json_file: + with open(json_file_path, 'r') as json_file: print("Processing file: {}".format(json_file_path)) # Grab data json_content = json_file.read() @@ -307,6 +349,10 @@ def sliding_window(): regression_file[testid][f"{os_name}-{arch}-{context}-{io}-{tls}"] = compute_baseline(data, testid) watermark_regression_file[testid][f"{os_name}-{arch}-{context}-{io}-{tls}"] = compute_baseline_watermark(data, testid) + # Remap DB test IDs to consumer-format keys before saving + regression_file = remap_to_consumer_keys(regression_file) + watermark_regression_file = remap_to_consumer_keys(watermark_regression_file) + # Save results to a json file. with open('regression.json', 'w') as f: json.dump(regression_file, f, indent=4) From 2ca384534d708f229863d03e91b3904d5684226a Mon Sep 17 00:00:00 2001 From: esgama107 <157749018+esgama107@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:43:03 -0700 Subject: [PATCH 3/3] Fix file path handling in regression.py wrong assessment about path name and folder --- pipeline/regression.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipeline/regression.py b/pipeline/regression.py index 59e9c493c..2a1b19eac 100644 --- a/pipeline/regression.py +++ b/pipeline/regression.py @@ -120,10 +120,10 @@ def singular_datapoint_method(): watermark_regression_file = {} for json_file_path in glob.glob('*.json'): - if not os.path.exists(json_file_path): + if not os.path.exists(f"{json_file_path}/{json_file_path}"): continue - with open(json_file_path, 'r') as json_file: + with open(f"{json_file_path}/{json_file_path}", 'r') as json_file: print("Processing file: {}".format(json_file_path)) # Grab data json_content = json_file.read() @@ -368,4 +368,4 @@ def sliding_window(): elif args.featureint == 2: sliding_window() else: - print("Method not supported.") \ No newline at end of file + print("Method not supported.")