From fa748a691ebadb261d4227f6c6186c00ecc8257e Mon Sep 17 00:00:00 2001 From: Joshua Moody Date: Thu, 12 Feb 2026 17:33:05 +0100 Subject: [PATCH 01/18] patch to fix failing production builds (#59623) --- package-lock.json | 65 +++++++++++++---------------------------------- package.json | 2 +- 2 files changed, 18 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4686088d94d4..e96a75e8a662 100644 --- a/package-lock.json +++ b/package-lock.json @@ -97,6 +97,7 @@ "swr": "^2.2.5", "tcp-port-used": "1.0.2", "tsx": "^4.19.4", + "typescript": "^5.8.3", "unified": "^11.0.5", "unist-util-find": "^3.0.0", "unist-util-visit": "^5.0.0", @@ -175,7 +176,6 @@ "robots-parser": "^3.0.1", "sass": "^1.77.8", "start-server-and-test": "^2.0.11", - "typescript": "^5.8.3", "unist-util-remove": "^4.0.0", "unist-util-visit-parents": "6.0.1", "vitest": "^4.0.4", @@ -312,6 +312,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" @@ -497,6 +498,7 @@ "version": "7.23.3", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.3.tgz", "integrity": "sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ==", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -534,12 +536,14 @@ "node_modules/@babel/core/node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "peer": true }, "node_modules/@babel/core/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "peer": true, "bin": { "semver": "bin/semver.js" } @@ -573,6 +577,7 @@ "version": "7.22.15", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz", "integrity": "sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==", + "peer": true, "dependencies": { "@babel/compat-data": "^7.22.9", "@babel/helper-validator-option": "^7.22.15", @@ -588,6 +593,7 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "peer": true, "dependencies": { "yallist": "^3.0.2" } @@ -596,6 +602,7 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "peer": true, "bin": { "semver": "bin/semver.js" } @@ -603,7 +610,8 @@ "node_modules/@babel/helper-compilation-targets/node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "peer": true }, "node_modules/@babel/helper-environment-visitor": { "version": "7.22.20", @@ -651,6 +659,7 @@ "version": "7.23.3", "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz", "integrity": "sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==", + "peer": true, "dependencies": { "@babel/helper-environment-visitor": "^7.22.20", "@babel/helper-module-imports": "^7.22.15", @@ -677,6 +686,7 @@ "version": "7.22.5", "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz", "integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==", + "peer": true, "dependencies": { "@babel/types": "^7.22.5" }, @@ -715,6 +725,7 @@ "version": "7.22.15", "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz", "integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -723,6 +734,7 @@ "version": "7.26.10", "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz", "integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==", + "peer": true, "dependencies": { "@babel/template": "^7.26.9", "@babel/types": "^7.26.10" @@ -2639,7 +2651,6 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -2940,7 +2951,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -2973,7 +2983,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.1.tgz", "integrity": "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -3013,7 +3022,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3034,7 +3042,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3055,7 +3062,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3076,7 +3082,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3097,7 +3102,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3118,7 +3122,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3139,7 +3142,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3160,7 +3162,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3181,7 +3182,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3202,7 +3202,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3223,7 +3222,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3244,7 +3242,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3265,7 +3262,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3283,7 +3279,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", - "dev": true, "license": "Apache-2.0", "optional": true, "bin": { @@ -3312,7 +3307,6 @@ "integrity": "sha512-vSMYtL/zOcFpvJCW71Q/OEGQb7KYBPAdKh35WNSkaZA75JlAO8ED8UN6GUNTm3drWomcbcqRPFqQbLae8yBTdg==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "playwright": "1.56.1" }, @@ -4136,7 +4130,6 @@ "integrity": "sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -4298,7 +4291,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.20.tgz", "integrity": "sha512-IPaCZN7PShZK/3t6Q87pfTkRm6oLTd4vztyoj+cbHUF1g3FfVb2tFIL79uCRKEfv16AhqDMBywP2VW3KIZUvcg==", "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.0.2" @@ -4310,7 +4302,6 @@ "integrity": "sha512-nf22//wEbKXusP6E9pfOCDwFdHAX4u172eaJI4YkDRQEZiorm6KfYnSC2SWLDMVWUOWPERmJnN0ujeAfTBLvrw==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -4481,7 +4472,6 @@ "integrity": "sha512-pUXGCuHnnKw6PyYq93lLRiZm3vjuslIy7tus1lIQTYVK9bL8XBgJnCWm8a0KcTtHC84Yya1Q6rtll+duSMj0dg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.39.1", "@typescript-eslint/types": "8.39.1", @@ -5127,7 +5117,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5157,7 +5146,6 @@ "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -5723,7 +5711,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001733", "electron-to-chromium": "^1.5.199", @@ -6005,7 +5992,6 @@ "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", "license": "MIT", - "peer": true, "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", @@ -7277,7 +7263,6 @@ "integrity": "sha512-TS9bTNIryDzStCpJN93aC5VRSW3uTx9sClUn4B87pwiCaJh220otoI0X8mJKr+VcPtniMdN8GKjlwgWGUv5ZKA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -7339,7 +7324,6 @@ "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", "dev": true, "license": "MIT", - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -7615,7 +7599,6 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -8755,7 +8738,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -8811,6 +8793,7 @@ "node_modules/gensync": { "version": "1.0.0-beta.2", "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -9017,7 +9000,6 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.9.0.tgz", "integrity": "sha512-GGTKBX4SD7Wdb8mqeDLni2oaRGYQWjWHGKPQ24ZMnUtKfcsVoiv4uX8+LJr1K6U5VW2Lu1BwJnj7uiori0YtRw==", "dev": true, - "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -10390,7 +10372,6 @@ "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", "dev": true, - "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -10517,6 +10498,7 @@ "node_modules/json5": { "version": "2.2.3", "license": "MIT", + "peer": true, "bin": { "json5": "lib/cli.js" }, @@ -12724,7 +12706,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", - "dev": true, "license": "MIT", "optional": true }, @@ -13479,7 +13460,6 @@ "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "playwright-core": "cli.js" }, @@ -13543,7 +13523,6 @@ "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -13713,7 +13692,6 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -13734,7 +13712,6 @@ "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -14402,7 +14379,6 @@ "integrity": "sha512-d0NoFH4v6SjEK7BoX810Jsrhj7IQSYHAHLi/iSpgqKc7LaIDshFRlSg5LOymf9FqQhxEHs2W5ZQXlvy0KD45Uw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.0.2", @@ -15340,7 +15316,6 @@ "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-5.3.11.tgz", "integrity": "sha512-uuzIIfnVkagcVHv9nE0VPlHPSCmXIUGKfJ42LNjxCCTDTL5sgnJ8Z7GZBq0EnLYGln77tPpEpExt2+qa+cZqSw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-module-imports": "^7.0.0", "@babel/traverse": "^7.4.5", @@ -15599,7 +15574,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -15923,9 +15897,7 @@ "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", - "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -16280,7 +16252,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "napi-postinstall": "^0.2.2" }, @@ -16483,7 +16454,6 @@ "integrity": "sha512-ZWyE8YXEXqJrrSLvYgrRP7p62OziLW7xI5HYGWFzOvupfAlrLvURSzv/FyGyy0eidogEM3ujU+kUG1zuHgb6Ug==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -16592,7 +16562,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, diff --git a/package.json b/package.json index 520c2d9d8baa..716a3003a5f2 100644 --- a/package.json +++ b/package.json @@ -247,6 +247,7 @@ "swr": "^2.2.5", "tcp-port-used": "1.0.2", "tsx": "^4.19.4", + "typescript": "^5.8.3", "unified": "^11.0.5", "unist-util-find": "^3.0.0", "unist-util-visit": "^5.0.0", @@ -325,7 +326,6 @@ "robots-parser": "^3.0.1", "sass": "^1.77.8", "start-server-and-test": "^2.0.11", - "typescript": "^5.8.3", "unist-util-remove": "^4.0.0", "unist-util-visit-parents": "6.0.1", "vitest": "^4.0.4", From 95c4cf2f5cb2b38e4d699ed653638aa095c3360e Mon Sep 17 00:00:00 2001 From: Kevin Heis Date: Thu, 12 Feb 2026 08:33:24 -0800 Subject: [PATCH 02/18] Add Article API-based search indexing module (#59497) Co-authored-by: Evan Bonsignori --- package-lock.json | 27 +- package.json | 2 + src/search/scripts/scrape/README.md | 26 +- .../scrape/lib/build-records-from-api.ts | 488 +++++++++++++ .../scripts/scrape/lib/build-records.ts | 234 ------- src/search/scripts/scrape/lib/domwaiter.ts | 167 ----- .../lib/parse-page-sections-into-records.ts | 95 --- .../scrape/lib/scrape-into-index-json.ts | 2 +- src/search/scripts/scrape/types.ts | 7 + src/search/tests/build-records-from-api.ts | 641 ++++++++++++++++++ .../tests/parse-page-sections-into-records.ts | 131 ---- 11 files changed, 1186 insertions(+), 634 deletions(-) create mode 100644 src/search/scripts/scrape/lib/build-records-from-api.ts delete mode 100644 src/search/scripts/scrape/lib/build-records.ts delete mode 100644 src/search/scripts/scrape/lib/domwaiter.ts delete mode 100644 src/search/scripts/scrape/lib/parse-page-sections-into-records.ts create mode 100644 src/search/tests/build-records-from-api.ts delete mode 100644 src/search/tests/parse-page-sections-into-records.ts diff --git a/package-lock.json b/package-lock.json index e96a75e8a662..520c5e5cb78a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,9 +67,11 @@ "lowlight": "^3.3.0", "markdownlint-rule-helpers": "^0.25.0", "mdast-util-from-markdown": "^2.0.2", + "mdast-util-gfm": "^3.1.0", "mdast-util-to-hast": "^13.2.1", "mdast-util-to-markdown": "2.1.2", "mdast-util-to-string": "^4.0.0", + "micromark-extension-gfm": "^3.0.0", "next": "^16.1.5", "ora": "^9.0.0", "parse5": "7.1.2", @@ -2983,6 +2985,7 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.1.tgz", "integrity": "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -3022,6 +3025,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3042,6 +3046,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3062,6 +3067,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3082,6 +3088,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3102,6 +3109,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3122,6 +3130,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3142,6 +3151,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3162,6 +3172,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3182,6 +3193,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3202,6 +3214,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3222,6 +3235,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3242,6 +3256,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3262,6 +3277,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3279,6 +3295,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", + "dev": true, "license": "Apache-2.0", "optional": true, "bin": { @@ -8738,6 +8755,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -11221,9 +11239,10 @@ "integrity": "sha512-MFETx3tbTjE7Uk6vvnWINA/1iJ7LuMdO4fcq8UfF0pRbj01aGLduVvQcRyswuACJdpnHgg8E3rQLhaRdNEJS0w==" }, "node_modules/mdast-util-gfm": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.0.0.tgz", - "integrity": "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", @@ -11583,6 +11602,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", "dependencies": { "micromark-extension-gfm-autolink-literal": "^2.0.0", "micromark-extension-gfm-footnote": "^2.0.0", @@ -12706,6 +12726,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "dev": true, "license": "MIT", "optional": true }, diff --git a/package.json b/package.json index 716a3003a5f2..a612cf398cb7 100644 --- a/package.json +++ b/package.json @@ -217,9 +217,11 @@ "lowlight": "^3.3.0", "markdownlint-rule-helpers": "^0.25.0", "mdast-util-from-markdown": "^2.0.2", + "mdast-util-gfm": "^3.1.0", "mdast-util-to-hast": "^13.2.1", "mdast-util-to-markdown": "2.1.2", "mdast-util-to-string": "^4.0.0", + "micromark-extension-gfm": "^3.0.0", "next": "^16.1.5", "ora": "^9.0.0", "parse5": "7.1.2", diff --git a/src/search/scripts/scrape/README.md b/src/search/scripts/scrape/README.md index 538052f51b96..5434aba6d179 100644 --- a/src/search/scripts/scrape/README.md +++ b/src/search/scripts/scrape/README.md @@ -1,16 +1,36 @@ # Scraping for General Search -We need to scrape each page on the Docs site and use the data we scrape to index Elasticsearch. +We fetch each page's content via the Article API and use the structured data to index Elasticsearch. This replaced the previous approach of rendering full HTML pages and scraping them with cheerio. We currently only scrape for **general search** results. Autocomplete search data is generated from analytics events and GPT queries. +## How it works + +The scrape script starts by loading all indexable pages, then for each page it calls the Article API (`/api/article?pathname=`) on the local server. The API returns structured JSON with the page's title, intro, breadcrumbs, and markdown body. The markdown is parsed into an AST with GFM support (so tables are handled cleanly), navigational headings like "Further reading" are filtered out, and the full content (including code blocks) is converted to plain text for indexing. + +The implementation lives in `lib/build-records-from-api.ts`. + ## CLI Script -Before running the scraping script ensure that the server is running in another terminal with `npm run general-search-scrape-server` +Before running the scraping script, start the server in another terminal: + +```bash +npm run general-search-scrape-server +``` + +Then run the scrape: -Run the script with `npm run general-search-scrape -- ` +```bash +npm run general-search-scrape -- +``` + +To scrape a specific language and version: + +```bash +npx tsx src/search/scripts/scrape/scrape-cli.ts -l en -V fpt +``` After a successful run it will generate a series of JSON files with the page data of every page of the Docs site into the passed directory. diff --git a/src/search/scripts/scrape/lib/build-records-from-api.ts b/src/search/scripts/scrape/lib/build-records-from-api.ts new file mode 100644 index 000000000000..ea618f092531 --- /dev/null +++ b/src/search/scripts/scrape/lib/build-records-from-api.ts @@ -0,0 +1,488 @@ +/** + * Build search records using the Article API instead of HTML scraping. + * + * This module provides functions to fetch article content via the Article API + * and convert it to search index records. This approach is faster and more + * reliable than HTML scraping because it: + * - Fetches pre-rendered markdown directly (no full HTML rendering) + * - Uses structured metadata (title, intro, breadcrumbs) from API + * - Parses headings from markdown using mdast (proper AST parsing) + */ + +import Bottleneck from 'bottleneck' +import chalk from 'chalk' +import dotenv from 'dotenv' +import boxen from 'boxen' +import { fromMarkdown } from 'mdast-util-from-markdown' +import { toString } from 'mdast-util-to-string' +import { visit } from 'unist-util-visit' +import { gfm } from 'micromark-extension-gfm' +import { gfmFromMarkdown } from 'mdast-util-gfm' +import GithubSlugger from 'github-slugger' +import type { Node, Parent } from 'unist' + +import languages from '@/languages/lib/languages-server' +import getPopularPages from '@/search/scripts/scrape/lib/popular-pages' +import { getAllVersionsKeyFromIndexVersion } from '@/search/lib/elasticsearch-versions' +import { fetchWithRetry } from '@/frame/lib/fetch-utils' + +import type { + Record, + FailedPage, + Page, + Permalink, + Config, + Redirects, +} from '@/search/scripts/scrape/types' + +// Same ignored headings as the HTML scraping approach +const IGNORED_HEADING_SLUGS = new Set(['in-this-article', 'further-reading', 'prerequisites']) + +// Known translations of the 3 ignored navigational headings. +// These are used as a fallback when github-slugger produces non-ASCII slugs +// that don't match the English slug set above. +const IGNORED_HEADING_TEXTS = new Set([ + // English (lowercase) + 'in this article', + 'further reading', + 'prerequisites', + // Japanese (ja) + 'この記事の内容', + '参考資料', + '前提条件', + // Chinese (zh) + '本文内容', + '延伸阅读', + '先决条件', + // Korean (ko) + '이 문서의 내용', + '추가 참고 자료', + '필수 조건', + // Spanish (es) + 'en este artículo', + 'información adicional', + 'requisitos previos', + // Portuguese (pt) + 'neste artigo', + 'leitura adicional', + 'pré-requisitos', + // Russian (ru) + 'в этой статье', + 'дополнительные материалы', + 'необходимые компоненты', + // French (fr) + 'dans cet article', + 'pour aller plus loin', + 'prérequis', + // German (de) + 'in diesem artikel', + 'weiterführende themen', + 'voraussetzungen', +]) + +// Default port matches build-records.ts for consistency +const DEFAULT_PORT = 4002 + +dotenv.config() + +// These defaults are known to work fine in GitHub Actions. +const MAX_CONCURRENT = parseInt(process.env.BUILD_RECORDS_MAX_CONCURRENT || '5', 10) +const MIN_TIME = parseInt(process.env.BUILD_RECORDS_MIN_TIME || '200', 10) + +// These products forcibly get a popularity of 0 +const FORCE_0_POPULARITY_PRODUCTS = new Set(['contributing']) + +const pageMarker = chalk.green('|') +const recordMarker = chalk.grey('.') + +interface HeadingNode extends Node { + type: 'heading' + depth: number +} + +export interface ArticleApiResponse { + meta: { + title: string + intro: string + product: string + breadcrumbs?: Array<{ href: string; title: string }> + } + body: string +} + +export interface ArticleApiErrorResponse { + error: string +} + +export type ArticleApiResult = ArticleApiResponse | ArticleApiErrorResponse + +/** + * Parse markdown into an AST with GFM support (tables, strikethrough, etc.). + */ +function parseMarkdown(markdown: string) { + return fromMarkdown(markdown, { + extensions: [gfm()], + mdastExtensions: [gfmFromMarkdown()], + }) +} + +// Block container types whose children should be separated by newlines. +// These contain other block-level nodes (paragraphs, lists, etc.) and +// toString() would concatenate them without whitespace, producing tokens +// like "SSH.Make" that the ES tokenizer can't split. +const BLOCK_CONTAINER_TYPES = new Set([ + 'root', + 'blockquote', + 'list', + 'listItem', + 'table', + 'tableRow', + 'footnoteDefinition', +]) + +/** + * Convert an AST to plain text, joining block-level children with newlines. + * Recurses into block containers (lists, blockquotes, etc.) so that nested + * block boundaries also get whitespace — not just the root level. + */ +function astToPlainText(node: Node): string { + const parent = node as Parent + if (!parent.children) { + return toString(node) + } + + if (BLOCK_CONTAINER_TYPES.has(node.type)) { + return parent.children.map((child) => astToPlainText(child)).join('\n') + } + + // Leaf blocks (paragraph, heading, tableCell) and inline nodes: + // concatenate inline text directly. + return toString(node) +} + +/** + * Extract headings and plain-text content from markdown in a single AST pass. + * Headings are extracted first, then the full AST (including code blocks) + * is converted to plain text so that terms inside code examples remain + * searchable (e.g. `ssh_url`, `ssh://`). + */ +export function extractFromMarkdown(markdown: string): { headings: string; content: string } { + const ast = parseMarkdown(markdown) + + // 1. Extract h2 headings from the AST + const headings: string[] = [] + const slugger = new GithubSlugger() + + visit(ast, (node: Node) => { + if (node.type !== 'heading') return + const headingNode = node as HeadingNode + if (headingNode.depth !== 2) return + + const headingText = toString(node) + const slug = slugger.slug(headingText) + + // Skip navigational headings by slug or known translated text + if (IGNORED_HEADING_SLUGS.has(slug)) return + if (IGNORED_HEADING_TEXTS.has(headingText.toLowerCase().trim())) return + + headings.push(headingText) + }) + + // 2. Convert full AST to plain text (code blocks are kept so that terms + // appearing only in code examples remain searchable). + const content = astToPlainText(ast) + + return { headings: headings.join('\n'), content } +} + +/** + * Extract h2 headings from markdown content using mdast parser. + * Filters out navigational headings (in-this-article, further-reading, prerequisites). + */ +export function extractHeadingsFromMarkdown(markdown: string): string { + return extractFromMarkdown(markdown).headings +} + +/** + * Convert markdown to plain text for search indexing using mdast. + * This extracts all text content from the markdown AST, including code blocks. + */ +export function markdownToPlainText(markdown: string): string { + return extractFromMarkdown(markdown).content +} + +/** + * Convert Article API response to a search record. + */ +export function articleApiResponseToRecord(pathname: string, data: ArticleApiResponse): Record { + // Build breadcrumbs string (excluding the last one which is the current page) + const breadcrumbsArray = data.meta.breadcrumbs?.map((b) => b.title) || [] + const breadcrumbs = + breadcrumbsArray + .slice(0, breadcrumbsArray.length > 1 ? -1 : breadcrumbsArray.length) + .join(' / ') || '' + + // Single-pass extraction: parse markdown once to get both headings and content + const { headings, content: bodyText } = extractFromMarkdown(data.body) + + // Combine intro with body if intro isn't already in body + const intro = data.meta.intro || '' + const content = + intro && !bodyText.includes(intro.trim()) + ? `${intro.trim()}\n${bodyText.trim()}`.trim() + : bodyText.trim() + + return { + objectID: pathname, + breadcrumbs, + title: data.meta.title, + headings, + content, + intro, + toplevel: breadcrumbsArray[0] || '', + } +} + +export interface FetchResult { + record: Record | null + failure: FailedPage | null +} + +function isErrorResponse(data: ArticleApiResult): data is ArticleApiErrorResponse { + return 'error' in data +} + +/** + * Fetch article from API and convert to search record. + */ +export async function fetchArticleAsRecord( + pathname: string, + baseUrl: string = `http://localhost:${DEFAULT_PORT}`, +): Promise { + const url = `${baseUrl}/api/article?pathname=${encodeURIComponent(pathname)}` + + try { + const response = await fetchWithRetry(url, undefined, { + retries: 3, + throwHttpErrors: false, + timeout: 60000, + }) + if (!response.ok) { + let errorMessage = `HTTP ${response.status}: ${response.statusText}` + let errorType = `HTTP ${response.status}` + try { + const body = await response.json() + if (body && typeof body.error === 'string') { + errorMessage = body.error + errorType = 'API Error' + } + } catch { + /* ignore JSON parse errors */ + } + return { + record: null, + failure: { + url: pathname, + error: errorMessage, + errorType, + }, + } + } + + const data = (await response.json()) as ArticleApiResult + + // Check for error response (e.g., archived pages) + if (isErrorResponse(data)) { + return { + record: null, + failure: { + url: pathname, + error: data.error, + errorType: 'API Error', + }, + } + } + + const record = articleApiResponseToRecord(pathname, data) + return { record, failure: null } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const errorName = error instanceof Error ? error.name : undefined + const errorCode = (error as { code?: string }).code + + // Prefer structured timeout indicators (name/code), with a documented + // fallback to message inspection for environments that only expose text. + const isTimeout = + errorName === 'AbortError' || + errorCode === 'ETIMEDOUT' || + errorCode === 'ECONNABORTED' || + message.toLowerCase().includes('timeout') + + return { + record: null, + failure: { + url: pathname, + error: message, + errorType: isTimeout ? 'Timeout' : 'Network Error', + }, + } + } +} + +export interface BuildRecordsResult { + records: Record[] + failedPages: FailedPage[] +} + +/** + * Build search records for a given index using the Article API. + * This is a drop-in replacement for buildRecords from build-records.ts. + */ +export default async function buildRecordsFromApi( + indexName: string, + indexablePages: Page[], + indexVersion: string, + languageCode: string, + redirects: Redirects, + config: Config = {} as Config, +): Promise { + const pageVersion = getAllVersionsKeyFromIndexVersion(indexVersion) + const { noMarkers, docsInternalDataPath } = config + + console.log(`\n\nBuilding records for index '${indexName}' (${languages[languageCode].name})`) + + const records: Record[] = [] + const failedPages: FailedPage[] = [] + + // Filter pages for this language and version + const pages = indexablePages + .filter((page) => page.languageCode === languageCode) + .filter((page) => page.permalinks.some((permalink) => permalink.pageVersion === pageVersion)) + + // Get permalinks for this language and version, deduplicating by href. + // Cross-product children can cause the same page to appear multiple + // times in the tree under different parents. + const seen = new Set() + const permalinks = pages + .map((page) => + page.permalinks.find( + (permalink) => + permalink.languageCode === languageCode && permalink.pageVersion === pageVersion, + ), + ) + .filter((permalink): permalink is Permalink => { + if (!permalink) return false + if (seen.has(permalink.href)) return false + seen.add(permalink.href) + return true + }) + + const popularPages = docsInternalDataPath + ? await getPopularPages(docsInternalDataPath, redirects, indexVersion, languageCode) + : {} + + console.log('indexable pages', indexablePages.length) + console.log('pages in index', pages.length) + console.log('permalinks in index', permalinks.length) + console.log(pageMarker, 'denotes pages') + console.log(recordMarker, 'denotes records derived from sections of pages') + console.log('popular page ratios', Object.keys(popularPages).length) + + const hasPopularPages = Object.keys(popularPages).length > 0 + const baseUrl = `http://localhost:${DEFAULT_PORT}` + + // Use Bottleneck for rate limiting + const limiter = new Bottleneck({ + maxConcurrent: MAX_CONCURRENT, + minTime: MIN_TIME, + }) + + // Process all permalinks with rate limiting + const fetchPromises = permalinks.map((permalink) => + limiter.schedule(async () => { + const result = await fetchArticleAsRecord(permalink.href, baseUrl) + + if (result.failure) { + result.failure.relativePath = permalink.relativePath + failedPages.push(result.failure) + if (!noMarkers) process.stdout.write(chalk.red('✗')) + return null + } + + if (result.record) { + // Apply popularity + const pathArticle = permalink.relativePath.replace('/index.md', '').replace('.md', '') + let popularity = (hasPopularPages && popularPages[pathArticle]) || 0.0 + + if (FORCE_0_POPULARITY_PRODUCTS.size) { + const product = result.record.objectID.split('/')[2] + if (FORCE_0_POPULARITY_PRODUCTS.has(product)) { + popularity = 0.0 + } + } + + result.record.popularity = popularity + if (!noMarkers) process.stdout.write(pageMarker + recordMarker) + return result.record + } + + return null + }), + ) + + const results = await Promise.all(fetchPromises) + for (const record of results) { + if (record) records.push(record) + } + + console.log('\nrecords in index: ', records.length) + + // Report failed pages (same format as build-records.ts) + if (failedPages.length > 0) { + const failureCount = failedPages.length + const header = chalk.bold.red(`${failureCount} page(s) failed to scrape\n\n`) + + const failureList = failedPages + .slice(0, 10) + .map((failure, idx) => { + const number = chalk.gray(`${idx + 1}. `) + const errorType = chalk.yellow(failure.errorType) + const pathLine = failure.relativePath + ? `\n${chalk.cyan(' Path: ')}${failure.relativePath}` + : '' + const urlLine = failure.url ? `\n${chalk.cyan(' URL: ')}${failure.url}` : '' + const errorLine = `\n${chalk.gray(` Error: ${failure.error}`)}` + + return `${number}${errorType}${pathLine}${urlLine}${errorLine}` + }) + .join('\n\n') + + const remaining = + failureCount > 10 ? `\n\n${chalk.gray(`... and ${failureCount - 10} more`)}` : '' + + const boxContent = header + failureList + remaining + const box = boxen(boxContent, { + title: chalk.red('⚠ Failed Pages'), + padding: 1, + borderColor: 'yellow', + }) + + console.log(`\n${box}\n`) + + console.log( + chalk.yellow( + `💡 Tip: These failures won't stop the scraping process. The script will continue with the remaining pages.`, + ), + ) + + if (failedPages.some((f) => f.errorType === 'Timeout')) { + console.log( + chalk.gray( + ` For timeout errors, try: export BUILD_RECORDS_MAX_CONCURRENT=50 (currently ${MAX_CONCURRENT})`, + ), + ) + } + } + + return { records, failedPages } +} diff --git a/src/search/scripts/scrape/lib/build-records.ts b/src/search/scripts/scrape/lib/build-records.ts deleted file mode 100644 index d5a3b0336b2a..000000000000 --- a/src/search/scripts/scrape/lib/build-records.ts +++ /dev/null @@ -1,234 +0,0 @@ -import eventToPromise from 'event-to-promise' -import chalk from 'chalk' -import dotenv from 'dotenv' -import boxen from 'boxen' - -import languages from '@/languages/lib/languages-server' -import parsePageSectionsIntoRecords from '@/search/scripts/scrape/lib/parse-page-sections-into-records' -import getPopularPages from '@/search/scripts/scrape/lib/popular-pages' -import domwaiter from '@/search/scripts/scrape/lib/domwaiter' -import { getAllVersionsKeyFromIndexVersion } from '@/search/lib/elasticsearch-versions' - -import type { Page, Permalink, Record, Config, Redirects } from '@/search/scripts/scrape/types' - -// Custom error class to replace got's HTTPError -class HTTPError extends Error { - response: { ok: boolean; statusCode?: number } - request: { requestUrl?: { pathname?: string } } - - constructor( - message: string, - response: { ok: boolean; statusCode?: number }, - request: { requestUrl?: { pathname?: string } }, - ) { - super(message) - this.name = 'HTTPError' - this.response = response - this.request = request - } -} - -const pageMarker = chalk.green('|') -const recordMarker = chalk.grey('.') -const port = 4002 - -dotenv.config() - -// These defaults are known to work fine in GitHub Actions. -// For local development, you can override these in your local .env file. -// For example: -// echo 'BUILD_RECORDS_MAX_CONCURRENT=5' >> .env -// echo 'BUILD_RECORDS_MIN_TIME=200' >> .env -const MAX_CONCURRENT = parseInt(process.env.BUILD_RECORDS_MAX_CONCURRENT || '5', 10) -const MIN_TIME = parseInt(process.env.BUILD_RECORDS_MIN_TIME || '200', 10) - -// These products, forcibly always get a popularity of 0 independent of -// their actual popularity which comes from an external JSON file. -// The objective for this is to reduce their search result ranking -// when multiple docs match on a certain keyword(s). -const FORCE_0_POPULARITY_PRODUCTS = new Set(['contributing']) - -interface FailedPage { - url?: string - relativePath?: string - error: string - errorType: string -} - -export interface BuildRecordsResult { - records: Record[] - failedPages: FailedPage[] -} - -export default async function buildRecords( - indexName: string, - indexablePages: Page[], - indexVersion: string, - languageCode: string, - redirects: Redirects, - config: Config = {} as Config, -): Promise { - // Determine the page version from the index version - const pageVersion = getAllVersionsKeyFromIndexVersion(indexVersion) - - const { noMarkers, docsInternalDataPath } = config - console.log(`\n\nBuilding records for index '${indexName}' (${languages[languageCode].name})`) - const records: Record[] = [] - const pages = indexablePages - // exclude pages that are not in the current language - .filter((page) => page.languageCode === languageCode) - // exclude pages that don't have a permalink for the current product version - .filter((page) => page.permalinks.some((permalink) => permalink.pageVersion === pageVersion)) - - // Find the approve permalink for the given language and GitHub product variant (dotcom v enterprise) - const permalinks = pages - .map((page) => { - return page.permalinks.find((permalink) => { - return permalink.languageCode === languageCode && permalink.pageVersion === pageVersion - }) - }) - .map((permalink) => { - if (permalink) { - permalink.url = `http://localhost:${port}${permalink.href}` - } - return permalink - }) - .filter((permalink): permalink is Permalink => permalink !== undefined) - - const popularPages = docsInternalDataPath - ? await getPopularPages(docsInternalDataPath, redirects, indexVersion, languageCode) - : {} - - console.log('indexable pages', indexablePages.length) - console.log('pages in index', pages.length) - console.log('permalinks in index', permalinks.length) - console.log(pageMarker, 'denotes pages') - console.log(recordMarker, 'denotes records derived from sections of pages') - console.log('popular page ratios', Object.keys(popularPages).length) - - const hasPopularPages = Object.keys(popularPages).length > 0 - - // Track failed pages - const failedPages: FailedPage[] = [] - - const waiter = domwaiter(permalinks, { maxConcurrent: MAX_CONCURRENT, minTime: MIN_TIME }) - .on('page', (page) => { - if (!noMarkers) process.stdout.write(pageMarker) - const newRecord = parsePageSectionsIntoRecords(page) - const pathArticle = page.relativePath.replace('/index.md', '').replace('.md', '') - let popularity = (hasPopularPages && popularPages[pathArticle]) || 0.0 - if (FORCE_0_POPULARITY_PRODUCTS.size) { - const product = newRecord.objectID.split('/')[2] - if (FORCE_0_POPULARITY_PRODUCTS.has(product)) { - popularity = 0.0 - } - } - newRecord.popularity = popularity - - if (!noMarkers) process.stdout.write(recordMarker) - records.push(newRecord) - }) - .on('error', (err) => { - // Track the failure - const url = (err as unknown as { url?: string }).url - const relativePath = (err as unknown as { relativePath?: string }).relativePath - - // Check for HTTPError by name since it may come from a different module - if ( - (err instanceof HTTPError || err?.name === 'HTTPError') && - (err as unknown as HTTPError).response - ) { - const httpErr = err as unknown as HTTPError - failedPages.push({ - url: httpErr.request?.requestUrl?.pathname || url, - relativePath, - error: err.message, - errorType: `HTTP ${httpErr.response?.statusCode || 'Error'}`, - }) - - if (!noMarkers) process.stdout.write(chalk.red('✗')) - } else if (err instanceof Error) { - // Enhanced error handling for timeout and network errors - const errorType = (err.cause as unknown as { code?: string })?.code || err.name - const isTimeout = - errorType === 'UND_ERR_HEADERS_TIMEOUT' || - errorType === 'UND_ERR_CONNECT_TIMEOUT' || - err.message.includes('timed out') - - failedPages.push({ - url, - relativePath, - error: err.message, - errorType: isTimeout ? 'Timeout' : errorType || 'Unknown Error', - }) - - if (!noMarkers) process.stdout.write(chalk.red('✗')) - } else { - failedPages.push({ - url, - relativePath, - error: String(err), - errorType: 'Unknown Error', - }) - - if (!noMarkers) process.stdout.write(chalk.red('✗')) - } - }) - - // Wait for 'done' event but ignore 'error' events (they're handled by the error listener above) - await eventToPromise(waiter, 'done', { ignoreErrors: true }) - console.log('\nrecords in index: ', records.length) - - // Report failed pages if any - if (failedPages.length > 0) { - const failureCount = failedPages.length - const header = chalk.bold.red(`${failureCount} page(s) failed to scrape\n\n`) - - const failureList = failedPages - .slice(0, 10) // Show first 10 failures - .map((failure, idx) => { - const number = chalk.gray(`${idx + 1}. `) - const errorType = chalk.yellow(failure.errorType) - const pathLine = failure.relativePath - ? `\n${chalk.cyan(' Path: ')}${failure.relativePath}` - : '' - const urlLine = failure.url ? `\n${chalk.cyan(' URL: ')}${failure.url}` : '' - const errorLine = `\n${chalk.gray(` Error: ${failure.error}`)}` - - return `${number}${errorType}${pathLine}${urlLine}${errorLine}` - }) - .join('\n\n') - - const remaining = - failureCount > 10 ? `\n\n${chalk.gray(`... and ${failureCount - 10} more`)}` : '' - - const boxContent = header + failureList + remaining - const box = boxen(boxContent, { - title: chalk.red('⚠ Failed Pages'), - padding: 1, - borderColor: 'yellow', - }) - - console.log(`\n${box}\n`) - - // Log suggestion - console.log( - chalk.yellow( - `💡 Tip: These failures won't stop the scraping process. The script will continue with the remaining pages.`, - ), - ) - - if (failedPages.some((f) => f.errorType === 'Timeout')) { - console.log( - chalk.gray( - ` For timeout errors, try: export BUILD_RECORDS_MAX_CONCURRENT=50 (currently ${MAX_CONCURRENT})`, - ), - ) - } - } - - return { - records, - failedPages, - } -} diff --git a/src/search/scripts/scrape/lib/domwaiter.ts b/src/search/scripts/scrape/lib/domwaiter.ts deleted file mode 100644 index 70e1251f6fe0..000000000000 --- a/src/search/scripts/scrape/lib/domwaiter.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { EventEmitter } from 'events' -import Bottleneck from 'bottleneck' -import { fetchWithRetry } from '@/frame/lib/fetch-utils' -import cheerio from 'cheerio' - -import type { Permalink } from '@/search/scripts/scrape/types' - -// Custom error class to match got's HTTPError interface -class HTTPError extends Error { - response: { ok: boolean; statusCode?: number } - request: { requestUrl?: { pathname?: string } } - - constructor( - message: string, - response: { ok: boolean; statusCode?: number }, - request: { requestUrl?: { pathname?: string } }, - ) { - super(message) - this.name = 'HTTPError' - this.response = response - this.request = request - } -} - -// Type aliases for error objects with additional URL information -type HTTPErrorWithUrl = HTTPError & { url?: string; relativePath?: string } -type ErrorWithUrl = Error & { url?: string; relativePath?: string } - -interface DomWaiterOptions { - parseDOM?: boolean - json?: boolean - maxConcurrent?: number - minTime?: number -} - -export default function domwaiter(pages: Permalink[], opts: DomWaiterOptions = {}): EventEmitter { - const emitter = new EventEmitter() - - // Add a default no-op error handler to prevent EventEmitter from throwing - // when errors are emitted before the caller attaches their error handler - // This will be overridden/supplemented by the caller's error handler - const defaultErrorHandler = () => { - // No-op: prevents EventEmitter from throwing - // External handlers will still receive the error - } - emitter.on('error', defaultErrorHandler) - - const defaults = { - parseDOM: true, - json: false, - maxConcurrent: 5, - minTime: 500, - } - opts = Object.assign(defaults, opts) - - const limiter = new Bottleneck(opts) - - for (const page of pages) { - async function schedulePage() { - try { - await limiter.schedule(() => getPage(page, emitter, opts)) - } catch (err) { - // Catch any unhandled promise rejections - emitter.emit('error', err) - } - } - - schedulePage() - } - - limiter.on('idle', () => { - emitter.emit('done') - }) - - limiter.on('error', (err) => { - emitter.emit('error', err) - }) - - return emitter -} - -async function getPage(page: Permalink, emitter: EventEmitter, opts: DomWaiterOptions) { - // Wrap everything in a try-catch to ensure no errors escape - try { - emitter.emit('beforePageLoad', page) - - if (opts.json) { - try { - const response = await fetchWithRetry(page.url!, undefined, { - retries: 3, - throwHttpErrors: false, - timeout: 60000, - }) - if (!response.ok) { - const httpError = new HTTPError( - `HTTP ${response.status}: ${response.statusText}`, - { ok: response.ok, statusCode: response.status }, - { requestUrl: { pathname: page.url } }, - ) - // Add URL and path info directly to the HTTPError - ;(httpError as HTTPErrorWithUrl).url = page.url - ;(httpError as HTTPErrorWithUrl).relativePath = page.relativePath - // Emit error instead of throwing - emitter.emit('error', httpError) - return // Exit early, don't continue processing - } - const json = await response.json() - const pageCopy = Object.assign({}, page, { json }) - emitter.emit('page', pageCopy) - } catch (err) { - // Enhance error with URL information - if (err instanceof Error && page.url) { - const enhancedError = new Error(err.message, { cause: err.cause }) - enhancedError.name = err.name - enhancedError.stack = err.stack - ;(enhancedError as ErrorWithUrl).url = page.url - ;(enhancedError as ErrorWithUrl).relativePath = page.relativePath - emitter.emit('error', enhancedError) - } else { - emitter.emit('error', err) - } - } - } else { - try { - const response = await fetchWithRetry(page.url!, undefined, { - retries: 3, - throwHttpErrors: false, - timeout: 60000, - }) - if (!response.ok) { - const httpError = new HTTPError( - `HTTP ${response.status}: ${response.statusText}`, - { ok: response.ok, statusCode: response.status }, - { requestUrl: { pathname: page.url } }, - ) - // Add URL and path info directly to the HTTPError - ;(httpError as HTTPErrorWithUrl).url = page.url - ;(httpError as HTTPErrorWithUrl).relativePath = page.relativePath - // Emit error instead of throwing - emitter.emit('error', httpError) - return // Exit early, don't continue processing - } - const body = await response.text() - const pageCopy = Object.assign({}, page, { body }) - if (opts.parseDOM) - (pageCopy as Permalink & { $?: ReturnType }).$ = cheerio.load(body) - emitter.emit('page', pageCopy) - } catch (err) { - // Enhance error with URL information - if (err instanceof Error && page.url) { - const enhancedError = new Error(err.message, { cause: err.cause }) - enhancedError.name = err.name - enhancedError.stack = err.stack - ;(enhancedError as ErrorWithUrl).url = page.url - ;(enhancedError as ErrorWithUrl).relativePath = page.relativePath - emitter.emit('error', enhancedError) - } else { - emitter.emit('error', err) - } - } - } - } catch (err) { - // Ultimate catch-all to ensure nothing escapes - console.error('Unexpected error in getPage:', err) - emitter.emit('error', err) - } -} diff --git a/src/search/scripts/scrape/lib/parse-page-sections-into-records.ts b/src/search/scripts/scrape/lib/parse-page-sections-into-records.ts deleted file mode 100644 index c862ad5d2464..000000000000 --- a/src/search/scripts/scrape/lib/parse-page-sections-into-records.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { render } from 'cheerio-to-text' - -import type { Record } from '@/search/scripts/scrape/types' - -// This module takes cheerio page object and divides it into sections -// using H1,H2 heading elements as section delimiters. The text -// that follows each heading becomes the content of the search record. - -const ignoredHeadingSlugs = ['in-this-article', 'further-reading', 'prerequisites'] - -export default function parsePageSectionsIntoRecords(page: any): Record { - const { href, $ } = page - const title = $('h1').first().text().trim() - const breadcrumbsArray = $('[data-search=breadcrumbs] nav.breadcrumbs a') - .map((i: number, el: any) => { - return $(el).text().trim().replace('/', '').replace(/\s+/g, ' ') - }) - .get() - - // Like in printing from DOM, some elements should not be included in - // the records for search. This might be navigational elements of the - // page that don't make much sense to find in a site search. - $('[data-search=hide]').remove() - - // Only slice off the last one if the length of the array is greater than 1 - // On an article page, we the breadcrumbs array will be something - // like: - // - // ['Product short title', 'Subcategory', 'Article title'] - // - // But on a product landing page, it'll just be: - // - // ['Product short title'] - // - // So here, if we skip the last one we get nothing for the breadcrumb. - const breadcrumbs = - breadcrumbsArray - .slice(0, breadcrumbsArray.length > 1 ? -1 : breadcrumbsArray.length) - .join(' / ') || '' - - const toplevel = breadcrumbsArray[0] || '' - const objectID = href - - const rootSelector = '[data-search=article-body]' - const $root = $(rootSelector) - if ($root.length === 0) { - console.warn(`${href} has no '${rootSelector}'`) - } else if ($root.length > 1) { - console.warn(`${href} has more than one '${rootSelector}' (${$root.length})`) - } - - const $sections = $('h2', $root) - .filter('[id]') - .filter((i: number, el: any) => { - return !ignoredHeadingSlugs.includes($(el).attr('id')) - }) - - const headings = $sections - .map((i: number, el: any) => $(el).text()) - .get() - .join('\n') - .trim() - - const intro = $('[data-search=lead] p').text().trim() - - let body = '' - // Typical example pages with no `$root` are: - // https://docs.github.com/en/code-security/guides - // - // We need to avoid these because if you use `getAllText()` on these - // pages, it will extract *everything* from the page, which will - // include the side bar and footer. - // Note: We're not adding custom extraction for guide pages as they are - // being phased out and don't warrant the effort. - if ($root.length > 0) { - body = render($root) - } - - if (!body && !intro) { - console.warn(`${objectID} has no body and no intro.`) - } - - const content = - intro && !body.includes(intro.trim()) ? `${intro.trim()}\n${body.trim()}`.trim() : body.trim() - - return { - objectID, - breadcrumbs, - title, - headings, - content, - intro, - toplevel, - } -} diff --git a/src/search/scripts/scrape/lib/scrape-into-index-json.ts b/src/search/scripts/scrape/lib/scrape-into-index-json.ts index 1c764d74a563..b7d50b21423e 100644 --- a/src/search/scripts/scrape/lib/scrape-into-index-json.ts +++ b/src/search/scripts/scrape/lib/scrape-into-index-json.ts @@ -1,7 +1,7 @@ import chalk from 'chalk' import languages from '@/languages/lib/languages-server' -import buildRecords from '@/search/scripts/scrape/lib/build-records' +import buildRecords from '@/search/scripts/scrape/lib/build-records-from-api' import findIndexablePages from '@/search/scripts/scrape/lib/find-indexable-pages' import { writeIndexRecords } from '@/search/scripts/scrape/lib/search-index-records' import { getElasticSearchIndex } from '@/search/lib/elasticsearch-indexes' diff --git a/src/search/scripts/scrape/types.ts b/src/search/scripts/scrape/types.ts index 20db4d78b968..96169a695dbd 100644 --- a/src/search/scripts/scrape/types.ts +++ b/src/search/scripts/scrape/types.ts @@ -68,3 +68,10 @@ export interface Redirects { export interface PopularPages { [key: string]: number } + +export interface FailedPage { + url?: string + relativePath?: string + error: string + errorType: string +} diff --git a/src/search/tests/build-records-from-api.ts b/src/search/tests/build-records-from-api.ts new file mode 100644 index 000000000000..b8b333f327df --- /dev/null +++ b/src/search/tests/build-records-from-api.ts @@ -0,0 +1,641 @@ +import { describe, expect, test, vi, beforeEach, afterEach } from 'vitest' + +import { + extractHeadingsFromMarkdown, + extractFromMarkdown, + markdownToPlainText, + articleApiResponseToRecord, + fetchArticleAsRecord, + type ArticleApiResponse, +} from '@/search/scripts/scrape/lib/build-records-from-api' + +import { fetchWithRetry } from '@/frame/lib/fetch-utils' + +vi.mock('@/frame/lib/fetch-utils', () => ({ + fetchWithRetry: vi.fn(), +})) + +const mockFetchWithRetry = vi.mocked(fetchWithRetry) + +describe('extractHeadingsFromMarkdown', () => { + test('extracts h2 headings', () => { + const markdown = `# Title + +Some intro text. + +## First Section + +Content here. + +## Second Section + +More content. + +### Subsection + +This should be ignored (h3). +` + const headings = extractHeadingsFromMarkdown(markdown) + expect(headings).toBe('First Section\nSecond Section') + }) + + test('filters out navigational headings', () => { + const markdown = `# Title + +## In this article + +Navigation links. + +## Main Content + +The actual content. + +## Further reading + +More links. + +## Prerequisites + +Setup steps. +` + const headings = extractHeadingsFromMarkdown(markdown) + expect(headings).toBe('Main Content') + }) + + test('handles markdown formatting in headings', () => { + const markdown = `# Title + +## Using \`code\` in headings + +## A [link](https://example.com) heading + +## **Bold** heading +` + const headings = extractHeadingsFromMarkdown(markdown) + // Verify complete heading text with formatting stripped + expect(headings).toContain('Using code in headings') + expect(headings).toContain('A link heading') + expect(headings).toContain('Bold heading') + // Should not contain markdown syntax + expect(headings).not.toContain('`') + expect(headings).not.toContain('**') + expect(headings).not.toContain('](') + }) + + test('returns empty string for no h2 headings', () => { + const markdown = `# Just a title + +Some content without sections. +` + const headings = extractHeadingsFromMarkdown(markdown) + expect(headings).toBe('') + }) + + test('filters out Japanese navigational headings', () => { + const markdown = `# タイトル + +## この記事の内容 + +ナビゲーション。 + +## メインコンテンツ + +実際の内容。 + +## 参考資料 + +リンク。 + +## 前提条件 + +セットアップ。 +` + const headings = extractHeadingsFromMarkdown(markdown) + expect(headings).toBe('メインコンテンツ') + }) + + test('filters out non-English navigational headings across languages', () => { + // Chinese + expect(extractHeadingsFromMarkdown('## 本文内容\n\n## 实际内容')).toBe('实际内容') + expect(extractHeadingsFromMarkdown('## 延伸阅读\n\n## 实际内容')).toBe('实际内容') + + // Korean + expect(extractHeadingsFromMarkdown('## 이 문서의 내용\n\n## 실제 내용')).toBe('실제 내용') + expect(extractHeadingsFromMarkdown('## 추가 참고 자료\n\n## 실제 내용')).toBe('실제 내용') + + // Spanish + expect(extractHeadingsFromMarkdown('## En este artículo\n\n## Contenido real')).toBe( + 'Contenido real', + ) + expect(extractHeadingsFromMarkdown('## Información adicional\n\n## Contenido real')).toBe( + 'Contenido real', + ) + + // French + expect(extractHeadingsFromMarkdown('## Dans cet article\n\n## Contenu réel')).toBe( + 'Contenu réel', + ) + expect(extractHeadingsFromMarkdown('## Prérequis\n\n## Contenu réel')).toBe('Contenu réel') + + // German + expect(extractHeadingsFromMarkdown('## Voraussetzungen\n\n## Echter Inhalt')).toBe( + 'Echter Inhalt', + ) + }) +}) + +describe('markdownToPlainText', () => { + test('converts markdown to plain text', () => { + const markdown = `# Title + +This is **bold** and *italic* text. + +- List item 1 +- List item 2 + +[A link](https://example.com) +` + const text = markdownToPlainText(markdown) + expect(text).toContain('Title') + expect(text).toContain('bold') + expect(text).toContain('italic') + expect(text).toContain('List item 1') + expect(text).toContain('A link') + // Should not contain markdown syntax + expect(text).not.toContain('**') + expect(text).not.toContain('](') + }) + + test('includes fenced code block content', () => { + const markdown = `Some text. + +\`\`\`javascript +const x = 1; +\`\`\` + +More text. +` + const text = markdownToPlainText(markdown) + expect(text).toContain('Some text') + expect(text).toContain('More text') + expect(text).toContain('const x = 1') + }) + + test('preserves inline code', () => { + const markdown = 'Use the `git commit` command to save changes.' + const text = markdownToPlainText(markdown) + expect(text).toContain('git commit') + expect(text).toContain('Use the') + }) + + test('inserts whitespace between list items', () => { + const markdown = `1. First item ends with SSH. + +2. Make a request using the CLI. +` + const text = markdownToPlainText(markdown) + // "SSH." and "Make" must not merge into "SSH.Make" + expect(text).not.toMatch(/SSH\.Make/) + expect(text).toMatch(/SSH\.\n/) + expect(text).toContain('Make a request') + }) + + test('inserts whitespace between nested block elements', () => { + const markdown = `> First paragraph in blockquote. +> +> Second paragraph in blockquote. +` + const text = markdownToPlainText(markdown) + // Paragraphs within a blockquote should be separated + expect(text).not.toMatch(/blockquote\.Second/) + expect(text).toContain('First paragraph in blockquote.') + expect(text).toContain('Second paragraph in blockquote.') + }) + + test('handles GFM tables cleanly', () => { + const markdown = `Some intro. + +| Column A | Column B | +| --- | --- | +| Cell 1 | Cell 2 | +| Cell 3 | Cell 4 | + +More text. +` + const text = markdownToPlainText(markdown) + expect(text).toContain('Column A') + expect(text).toContain('Cell 1') + expect(text).toContain('More text') + // Should not contain raw GFM table syntax artifacts + expect(text).not.toContain('| ---') + expect(text).not.toContain('---') + }) +}) + +describe('extractFromMarkdown', () => { + test('returns both headings and content in a single pass', () => { + const markdown = `# Title + +## Section One + +Some content. + +## Further reading + +Links here. + +\`\`\`json +{ "key": "value" } +\`\`\` + +## Section Two + +More content. +` + const result = extractFromMarkdown(markdown) + + // Headings should exclude "Further reading" + expect(result.headings).toBe('Section One\nSection Two') + + // Content should include fenced code block text + expect(result.content).toContain('Some content') + expect(result.content).toContain('More content') + expect(result.content).toContain('"key"') + }) + + test('produces same results as separate wrapper calls', () => { + const markdown = `# Test + +## Heading One + +Body text here. + +## Prerequisites + +Setup info. +` + const combined = extractFromMarkdown(markdown) + const headingsOnly = extractHeadingsFromMarkdown(markdown) + const contentOnly = markdownToPlainText(markdown) + + expect(combined.headings).toBe(headingsOnly) + expect(combined.content).toBe(contentOnly) + }) +}) + +describe('articleApiResponseToRecord', () => { + test('converts API response to search record', () => { + const response: ArticleApiResponse = { + meta: { + title: 'About GitHub', + intro: 'Learn about GitHub.', + product: 'Get started', + breadcrumbs: [ + { href: '/en/get-started', title: 'Get started' }, + { href: '/en/get-started/overview', title: 'Overview' }, + { href: '/en/get-started/overview/about-github', title: 'About GitHub' }, + ], + }, + body: `# About GitHub + +Learn about GitHub. + +## What is GitHub? + +GitHub is a platform for collaboration. + +## Getting started + +Here's how to begin. +`, + } + + const record = articleApiResponseToRecord('/en/get-started/overview/about-github', response) + + expect(record.objectID).toBe('/en/get-started/overview/about-github') + expect(record.title).toBe('About GitHub') + expect(record.intro).toBe('Learn about GitHub.') + expect(record.breadcrumbs).toBe('Get started / Overview') + expect(record.toplevel).toBe('Get started') + expect(record.headings).toBe('What is GitHub?\nGetting started') + expect(record.content).toContain('GitHub is a platform') + }) + + test('handles missing breadcrumbs (archived pages)', () => { + const response: ArticleApiResponse = { + meta: { + title: 'Archived Page', + intro: 'This is archived.', + product: 'Old product', + // No breadcrumbs - simulating archived page + }, + body: '# Archived Page\n\nContent here.', + } + + const record = articleApiResponseToRecord('/en/archived/page', response) + + expect(record.breadcrumbs).toBe('') + expect(record.toplevel).toBe('') + expect(record.title).toBe('Archived Page') + }) + + test('handles single breadcrumb (product landing page)', () => { + const response: ArticleApiResponse = { + meta: { + title: 'Get started', + intro: 'Welcome to GitHub.', + product: 'Get started', + breadcrumbs: [{ href: '/en/get-started', title: 'Get started' }], + }, + body: '# Get started\n\nWelcome.', + } + + const record = articleApiResponseToRecord('/en/get-started', response) + + // For single breadcrumb, don't slice it off + expect(record.breadcrumbs).toBe('Get started') + expect(record.toplevel).toBe('Get started') + }) + + test('prepends intro to content if not already present', () => { + const response: ArticleApiResponse = { + meta: { + title: 'Test', + intro: 'Unique intro text.', + product: 'Test', + breadcrumbs: [], + }, + body: '# Test\n\nDifferent body content.', + } + + const record = articleApiResponseToRecord('/en/test', response) + + expect(record.content).toMatch(/^Unique intro text\./) + expect(record.content).toContain('Different body content') + }) + + test('does not duplicate intro if already in body', () => { + const response: ArticleApiResponse = { + meta: { + title: 'Test', + intro: 'Same intro.', + product: 'Test', + breadcrumbs: [], + }, + body: '# Test\n\nSame intro.\n\nMore content.', + } + + const record = articleApiResponseToRecord('/en/test', response) + + // Intro should appear only once + const introCount = (record.content.match(/Same intro/g) || []).length + expect(introCount).toBe(1) + }) + + test('includes fenced code block content for search', () => { + const response: ArticleApiResponse = { + meta: { + title: 'REST API', + intro: 'API reference.', + product: 'REST', + breadcrumbs: [], + }, + body: `# REST API + +## Endpoints + +Use the endpoint below. + +\`\`\`json +{ + "ssh_url": "ssh://git@github.com/owner/repo.git", + "properties": { "name": "string" } +} +\`\`\` + +## Parameters + +The \`name\` parameter is required. +`, + } + + const record = articleApiResponseToRecord('/en/rest/api', response) + + expect(record.content).toContain('Use the endpoint below') + expect(record.content).toContain('parameter is required') + // Fenced code block content should be included for search + expect(record.content).toContain('ssh_url') + expect(record.content).toContain('ssh://git@github.com') + // Inline code content should also be preserved + expect(record.content).toContain('name') + }) +}) + +describe('fetchArticleAsRecord', () => { + beforeEach(() => { + vi.restoreAllMocks() + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + test('returns record on successful API response', async () => { + const mockResponse: ArticleApiResponse = { + meta: { + title: 'Test Article', + intro: 'Test intro.', + product: 'Test Product', + breadcrumbs: [{ href: '/en/test', title: 'Test' }], + }, + body: '# Test Article\n\nTest content.', + } + + mockFetchWithRetry.mockResolvedValue({ + ok: true, + json: () => Promise.resolve(mockResponse), + } as Response) + + const result = await fetchArticleAsRecord('/en/test/article', 'http://localhost:4002') + + expect(result.record).not.toBeNull() + expect(result.failure).toBeNull() + expect(result.record?.title).toBe('Test Article') + expect(result.record?.objectID).toBe('/en/test/article') + }) + + test('calls fetchWithRetry with correct options', async () => { + const mockResponse: ArticleApiResponse = { + meta: { + title: 'Test', + intro: 'Intro', + product: 'Product', + breadcrumbs: [], + }, + body: '# Test', + } + + mockFetchWithRetry.mockResolvedValue({ + ok: true, + json: () => Promise.resolve(mockResponse), + } as Response) + + await fetchArticleAsRecord('/en/test', 'http://localhost:4002') + + expect(mockFetchWithRetry).toHaveBeenCalledWith( + 'http://localhost:4002/api/article?pathname=%2Fen%2Ftest', + undefined, + { + retries: 3, + throwHttpErrors: false, + timeout: 60000, + }, + ) + }) + + test('returns failure for HTTP 404', async () => { + mockFetchWithRetry.mockResolvedValue({ + ok: false, + status: 404, + statusText: 'Not Found', + json: () => Promise.reject(new Error('no body')), + } as unknown as Response) + + const result = await fetchArticleAsRecord('/en/nonexistent', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('HTTP 404') + expect(result.failure?.error).toContain('404') + }) + + test('returns failure for HTTP 500', async () => { + mockFetchWithRetry.mockResolvedValue({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + json: () => Promise.reject(new Error('no body')), + } as unknown as Response) + + const result = await fetchArticleAsRecord('/en/broken', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('HTTP 500') + }) + + test('parses 403 response body for error message', async () => { + mockFetchWithRetry.mockResolvedValue({ + ok: false, + status: 403, + statusText: 'Forbidden', + json: () => Promise.resolve({ error: 'Page is archived and not available' }), + } as unknown as Response) + + const result = await fetchArticleAsRecord('/en/archived/page', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('API Error') + expect(result.failure?.error).toBe('Page is archived and not available') + }) + + test('falls back to HTTP status when 403 body is not JSON', async () => { + mockFetchWithRetry.mockResolvedValue({ + ok: false, + status: 403, + statusText: 'Forbidden', + json: () => Promise.reject(new Error('Invalid JSON')), + } as unknown as Response) + + const result = await fetchArticleAsRecord('/en/forbidden', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('HTTP 403') + expect(result.failure?.error).toBe('HTTP 403: Forbidden') + }) + + test('returns failure for API error response (archived pages)', async () => { + mockFetchWithRetry.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ error: 'This page is archived' }), + } as Response) + + const result = await fetchArticleAsRecord('/en/archived/page', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('API Error') + expect(result.failure?.error).toBe('This page is archived') + }) + + test('returns failure with Timeout errorType for AbortError', async () => { + const abortError = new Error('The operation was aborted') + abortError.name = 'AbortError' + + mockFetchWithRetry.mockRejectedValue(abortError) + + const result = await fetchArticleAsRecord('/en/slow/page', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('Timeout') + }) + + test('returns failure with Timeout errorType for ETIMEDOUT', async () => { + const timeoutError = new Error('Connection timed out') as Error & { code: string } + timeoutError.code = 'ETIMEDOUT' + + mockFetchWithRetry.mockRejectedValue(timeoutError) + + const result = await fetchArticleAsRecord('/en/slow/page', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('Timeout') + }) + + test('returns failure with Network Error for other errors', async () => { + mockFetchWithRetry.mockRejectedValue(new Error('Connection refused')) + + const result = await fetchArticleAsRecord('/en/unreachable', 'http://localhost:4002') + + expect(result.record).toBeNull() + expect(result.failure).not.toBeNull() + expect(result.failure?.errorType).toBe('Network Error') + expect(result.failure?.error).toBe('Connection refused') + }) + + test('FetchResult structure matches expected shape', async () => { + const mockResponse: ArticleApiResponse = { + meta: { + title: 'Test', + intro: 'Intro', + product: 'Product', + breadcrumbs: [], + }, + body: '# Test', + } + + mockFetchWithRetry.mockResolvedValue({ + ok: true, + json: () => Promise.resolve(mockResponse), + } as Response) + + const result = await fetchArticleAsRecord('/en/test', 'http://localhost:4002') + + // Verify the shape of FetchResult + expect(result).toHaveProperty('record') + expect(result).toHaveProperty('failure') + + // When successful, record should have all expected fields + expect(result.record).toHaveProperty('objectID') + expect(result.record).toHaveProperty('title') + expect(result.record).toHaveProperty('intro') + expect(result.record).toHaveProperty('content') + expect(result.record).toHaveProperty('headings') + expect(result.record).toHaveProperty('breadcrumbs') + expect(result.record).toHaveProperty('toplevel') + }) +}) diff --git a/src/search/tests/parse-page-sections-into-records.ts b/src/search/tests/parse-page-sections-into-records.ts deleted file mode 100644 index 0e2e0741fc9d..000000000000 --- a/src/search/tests/parse-page-sections-into-records.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { fileURLToPath } from 'url' -import path from 'path' -import fs from 'fs/promises' - -import cheerio from 'cheerio' -import { describe, expect, test } from 'vitest' - -import parsePageSectionsIntoRecords from '@/search/scripts/scrape/lib/parse-page-sections-into-records' -import type { Record } from '@/search/scripts/scrape/types' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) - -// Define the shape of fixtures with explicit keys and string values -const fixtures: { - pageWithSections: string - pageWithoutSections: string - pageWithoutBody: string - pageMultipleH1s: string - pageHeadingParagraphNoWhitespace: string -} = { - pageWithSections: await fs.readFile( - path.join(__dirname, 'fixtures/page-with-sections.html'), - 'utf8', - ), - pageWithoutSections: await fs.readFile( - path.join(__dirname, 'fixtures/page-without-sections.html'), - 'utf8', - ), - pageWithoutBody: await fs.readFile( - path.join(__dirname, 'fixtures/page-without-body.html'), - 'utf8', - ), - pageMultipleH1s: await fs.readFile( - path.join(__dirname, 'fixtures/page-with-multiple-h1s.html'), - 'utf8', - ), - pageHeadingParagraphNoWhitespace: await fs.readFile( - path.join(__dirname, 'fixtures/page-with-heading-and-paragraph-no-whitespace.html'), - 'utf8', - ), -} - -describe('search parsePageSectionsIntoRecords module', () => { - test('works for pages with sections', () => { - const html: string = fixtures.pageWithSections - const $ = cheerio.load(html) - const href: string = '/example/href' - const record: Record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) - const expected: Record = { - objectID: '/example/href', - breadcrumbs: 'GitHub Actions / actions learning path', - title: 'I am the page title', - headings: 'First heading\nSecond heading\nTable heading', - content: - 'This is an introduction to the article.\n' + - "In this article\nThis won't be ignored.\nFirst heading\n" + - "Here's a paragraph.\nAnd another.\nSecond heading\n" + - "Here's a paragraph in the second section.\nAnd another.\n" + - 'Table heading\nPeter\nHuman\n' + - 'Bullet\nPoint\nNumbered\nList\n' + - "Further reading\nThis won't be ignored.", - intro: 'This is an introduction to the article.', - toplevel: 'GitHub Actions', - } - - expect(record).toEqual(expected) - }) - - test('works for pages without sections', () => { - const html: string = fixtures.pageWithoutSections - const $ = cheerio.load(html) - const href: string = '/example/href' - const record: Record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) - const expected: Record = { - objectID: '/example/href', - breadcrumbs: 'Education / subcategory', - title: 'A page without sections', - headings: '', - content: 'This is an introduction to the article.\nFirst paragraph.\nSecond paragraph.', - intro: 'This is an introduction to the article.', - toplevel: 'Education', - } - - expect(record).toEqual(expected) - }) - - test('works for pages without content', () => { - const html: string = fixtures.pageWithoutBody - const $ = cheerio.load(html) - const href: string = '/example/href' - const record: Record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) - const expected: Record = { - objectID: '/example/href', - breadcrumbs: 'Education / subcategory', - title: 'A page without body', - headings: '', - content: 'This is an introduction to the article.', - intro: 'This is an introduction to the article.', - toplevel: 'Education', - } - - expect(record).toEqual(expected) - }) - - test('only picks up the first h1 for the title', () => { - const html: string = fixtures.pageMultipleH1s - const $ = cheerio.load(html) - const href: string = '/example/href' - const record: Record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) - - expect(record.title).toEqual('I am the page title') - }) - - test("content doesn't lump headings with paragraphs together", () => { - const html: string = fixtures.pageHeadingParagraphNoWhitespace - const $ = cheerio.load(html) - const href: string = '/example/href' - const record: Record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) - - // Ensure the heading appears only once - const headingMatches = record.content.match(/Changing your primary email address/g) - expect(headingMatches).not.toBeNull() - expect(headingMatches!.length).toBe(1) - - // Ensure there's no concatenation without whitespace - expect(record.content.includes('email addressYou can set')).toBeFalsy() - - // Ensure inline elements remain intact - expect(record.content).toMatch(/Paragraph\./) - }) -}) From 90ffd24691127188cf2fb901de5b20ba3164680b Mon Sep 17 00:00:00 2001 From: Kevin Heis Date: Thu, 12 Feb 2026 09:32:16 -0800 Subject: [PATCH 03/18] feat: A/B test framework for content readability experiment (#59592) --- .../experiments/ExperimentContentSwap.tsx | 58 +++++++++++++++ .../experiments/content-ab-testing.md | 74 +++++++++++++++++++ .../components/experiments/experiment.ts | 2 + .../components/experiments/experiments.ts | 12 ++- src/frame/components/article/ArticlePage.tsx | 2 + 5 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 src/events/components/experiments/ExperimentContentSwap.tsx create mode 100644 src/events/components/experiments/content-ab-testing.md diff --git a/src/events/components/experiments/ExperimentContentSwap.tsx b/src/events/components/experiments/ExperimentContentSwap.tsx new file mode 100644 index 000000000000..f4703b5b4f8d --- /dev/null +++ b/src/events/components/experiments/ExperimentContentSwap.tsx @@ -0,0 +1,58 @@ +import { useLayoutEffect, useState } from 'react' +import { useShouldShowExperiment } from '@/events/components/experiments/useShouldShowExperiment' +import { EXPERIMENTS } from '@/events/components/experiments/experiments' + +const EXPERIMENT_KEY = EXPERIMENTS.readability_copilot.key + +// Swaps visibility of .exp-control / .exp-treatment divs within the article +// body based on the user's experiment group. Both variants are rendered +// server-side (and cached by Fastly). Treatment divs must have the `hidden` +// and `data-nosnippet` attributes in the authored HTML so control content is +// always the safe fallback and crawlers ignore the treatment variant. +export function ExperimentContentSwap({ containerRef }: { containerRef: string }) { + const [hasExperimentDivs, setHasExperimentDivs] = useState(false) + + // Check once on mount whether this page has any experiment markup. + // If not, skip the experiment hook entirely to avoid unnecessary work. + useLayoutEffect(() => { + const container = document.querySelector(containerRef) + if (container?.querySelector(`[data-experiment="${EXPERIMENT_KEY}"]`)) { + setHasExperimentDivs(true) + } + }, [containerRef]) + + if (!hasExperimentDivs) return null + + return +} + +// Separated so the experiment hook only runs when experiment divs are present. +function ExperimentSwapper({ containerRef }: { containerRef: string }) { + const { showExperiment, experimentLoading } = useShouldShowExperiment( + EXPERIMENTS.readability_copilot, + ) + + // useLayoutEffect fires synchronously after DOM mutations but before + // the browser paints, minimizing the flash of control content for + // treatment users. + useLayoutEffect(() => { + if (experimentLoading) return + + const container = document.querySelector(containerRef) + if (!container) return + + const selector = `[data-experiment="${EXPERIMENT_KEY}"]` + const controlDivs = container.querySelectorAll(`.exp-control${selector}`) + const treatmentDivs = container.querySelectorAll(`.exp-treatment${selector}`) + + if (showExperiment) { + for (const div of controlDivs) div.hidden = true + for (const div of treatmentDivs) div.hidden = false + } else { + for (const div of controlDivs) div.hidden = false + for (const div of treatmentDivs) div.hidden = true + } + }, [showExperiment, experimentLoading, containerRef]) + + return null +} diff --git a/src/events/components/experiments/content-ab-testing.md b/src/events/components/experiments/content-ab-testing.md new file mode 100644 index 000000000000..3fa33cbeabf4 --- /dev/null +++ b/src/events/components/experiments/content-ab-testing.md @@ -0,0 +1,74 @@ +# Content A/B testing + +This guide explains how to set up an A/B test on article content using the experiment framework. Both content variants live in the same Markdown file and are served in a single Fastly-cached response. Client-side JavaScript swaps which variant is visible based on the user's experiment group. + +For background on A/B testing at GitHub Docs, see [the A/B testing guide](https://github.com/github/docs-team/blob/main/analytics/ab-test.md). + +> [!IMPORTANT] +> Only one experiment can have `includeVariationInContext: true` at a time, because `experiment_variation` is a single key in the event context. Multiple experiments can run concurrently if the others use `sendExperimentSuccess` for tracking instead. See the [experiments README](./README.md) for details. + +## Authoring an experiment article + +Wrap the control (original) and treatment (rewritten) content in HTML divs. Both variants render as normal Markdown inside the divs. + +```markdown +--- +title: Your article title +--- + +
+ +Original article body here. Markdown renders normally inside the div. + +Links, lists, code blocks, and all other Markdown features work as usual. + +
+ +``` + +### Rules + +* **Blank lines required**: Leave a blank line after the opening `
` and before the closing `
` so Markdown renders correctly inside the div. +* **`data-experiment` attribute**: Must match the experiment key registered in `experiments.ts` (currently `readability_copilot`). +* **`hidden` attribute**: Always add `hidden` to the treatment div. This ensures the control is shown by default (safe fallback if JavaScript fails). +* **`data-nosnippet` attribute**: Always add `data-nosnippet` to the treatment div. This tells search engines to ignore the treatment text. +* **Multiple pairs**: You can have multiple control/treatment pairs in one article if only some sections differ. Each pair must have the matching `data-experiment`, class, and attributes. + +## Previewing the treatment + +* **Staff cookie**: If you have the `staffonly` cookie set, you will always see the treatment. +* **URL parameter**: Add `?feature=readability` to any article URL to force the treatment variant. +* **Console override**: In the browser console, run: + ```javascript + window.overrideControlGroup('readability_copilot', 'treatment') + ``` + Reload the page to see the treatment. Use `'control'` to switch back. + +## How tracking works + +When `includeVariationInContext` is `true` (which it is for this experiment), **every** analytics event on the page automatically includes `experiment_variation: "control"` or `"treatment"` in its context. This means: + +* Page view events → measure impressions per variant +* Link click events → measure CTA clicks per variant +* Exit events → measure scroll depth (`exit_scroll_length`), time on page (`exit_visit_duration`), and scroll engagement (`exit_scroll_flip`) per variant +* No extra tracking code is needed in the content + +CTA links that point to external sites (like `github.com/features/copilot`) are already tracked by the existing link event system. The `link_samesite: false` flag identifies external (CTA) clicks. + +To analyze additional event types (such as scroll depth or time on page), add queries to the dashboard—no code changes are needed. + +## Analyzing results + +Use the **[Docs Experiment Results dashboard](https://gh.io/docs-8c0c)** to track split verification, CTA click-through rates, sequential significance testing, and per-article breakdowns. The dashboard has parameters for experiment name, path product, and minimum detectable effect. + +Dashboard source config: [`docs-team/analytics/dashboard-builder/experiment-results.config.ts`](https://github.com/github/docs-team/blob/main/analytics/dashboard-builder/experiment-results.config.ts) + +## Ending the experiment + +1. Set `isActive: false` in `src/events/components/experiments/experiments.ts`. +2. Remove the experiment divs from the articles, keeping whichever variant won. +3. Open a PR documenting the results. diff --git a/src/events/components/experiments/experiment.ts b/src/events/components/experiments/experiment.ts index df264c710697..f7d30b2f7336 100644 --- a/src/events/components/experiments/experiment.ts +++ b/src/events/components/experiments/experiment.ts @@ -29,6 +29,8 @@ export function shouldShowExperiment( const experiments = getActiveExperiments('all') for (const experiment of experiments) { if (experiment.key === experimentKey) { + // Respect isActive so flipping it to false actually stops the experiment + if (!experiment.isActive) return false // If there is an override for the current session, use that if (controlGroupOverride[experiment.key]) { const controlGroup = getExperimentControlGroupFromSession( diff --git a/src/events/components/experiments/experiments.ts b/src/events/components/experiments/experiments.ts index 6e3e72ec67ff..f4fe2f607d5f 100644 --- a/src/events/components/experiments/experiments.ts +++ b/src/events/components/experiments/experiments.ts @@ -15,7 +15,7 @@ type Experiment = { } // Update this with the name of the experiment, e.g. | 'example_experiment' -export type ExperimentNames = 'placeholder_experiment' +export type ExperimentNames = 'placeholder_experiment' | 'readability_copilot' export const EXPERIMENTS = { // Placeholder experiment to maintain type compatibility @@ -29,6 +29,16 @@ export const EXPERIMENTS = { alwaysShowForStaff: false, turnOnWithURLParam: 'placeholder', // Placeholder URL param }, + readability_copilot: { + key: 'readability_copilot', + isActive: true, + percentOfUsersToGetExperiment: 50, + includeVariationInContext: true, + limitToLanguages: ['en'], + limitToVersions: [], + alwaysShowForStaff: true, + turnOnWithURLParam: 'readability', + }, /* Add new experiments here, example: 'example_experiment': { key: 'example_experiment', diff --git a/src/frame/components/article/ArticlePage.tsx b/src/frame/components/article/ArticlePage.tsx index 03b81abcfd01..daf49ffb7e30 100644 --- a/src/frame/components/article/ArticlePage.tsx +++ b/src/frame/components/article/ArticlePage.tsx @@ -24,6 +24,7 @@ import { LinkPreviewPopover } from '@/links/components/LinkPreviewPopover' import { UtmPreserver } from '@/frame/components/UtmPreserver' import { JourneyTrackCard, JourneyTrackNav } from '@/journeys/components' import { ViewMarkdownButton } from './ViewMarkdownButton' +import { ExperimentContentSwap } from '@/events/components/experiments/ExperimentContentSwap' const ClientSideRefresh = dynamic(() => import('@/frame/components/ClientSideRefresh'), { ssr: false, @@ -95,6 +96,7 @@ export const ArticlePage = () => { )} {renderedPage} + {effectiveDate && (
Effective as of:{' '} From cb779ca41af3534c05cef9357e3b25225ce338ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Feb 2026 09:43:21 -0800 Subject: [PATCH 04/18] Bump actions/download-artifact from 4.3.0 to 7.0.0 (#59597) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/index-general-search.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/index-general-search.yml b/.github/workflows/index-general-search.yml index e15c21140fb3..a06dacf3971f 100644 --- a/.github/workflows/index-general-search.yml +++ b/.github/workflows/index-general-search.yml @@ -254,7 +254,7 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Download all failure artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: pattern: search-failures-* path: /tmp/failures From 9b7358b9c8110818cb3b8fee20b122cc21570ad9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:44:51 +0000 Subject: [PATCH 05/18] Bump actions/upload-artifact from 4.6.2 to 6.0.0 (#59596) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/index-general-search.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/index-general-search.yml b/.github/workflows/index-general-search.yml index a06dacf3971f..cb9105855fea 100644 --- a/.github/workflows/index-general-search.yml +++ b/.github/workflows/index-general-search.yml @@ -232,7 +232,7 @@ jobs: - name: Upload failures artifact if: ${{ steps.check-failures.outputs.has_failures == 'true' }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: search-failures-${{ matrix.language }} path: /tmp/records/failures-summary.json From b5ca26d8ede3a5ce0d3836a6ada2411467f4512b Mon Sep 17 00:00:00 2001 From: docs-bot <77750099+docs-bot@users.noreply.github.com> Date: Thu, 12 Feb 2026 09:44:58 -0800 Subject: [PATCH 06/18] Delete orphaned files (2026-02-11-16-43) (#59603) --- .../dependabot/no-dependabot-alerts-for-malware.md | 1 - data/reusables/security-configurations/overview.md | 7 ------- 2 files changed, 8 deletions(-) delete mode 100644 data/reusables/dependabot/no-dependabot-alerts-for-malware.md delete mode 100644 data/reusables/security-configurations/overview.md diff --git a/data/reusables/dependabot/no-dependabot-alerts-for-malware.md b/data/reusables/dependabot/no-dependabot-alerts-for-malware.md deleted file mode 100644 index 8549af2f6e1a..000000000000 --- a/data/reusables/dependabot/no-dependabot-alerts-for-malware.md +++ /dev/null @@ -1 +0,0 @@ -{% data variables.product.prodname_dependabot %} doesn't generate {% data variables.product.prodname_dependabot_alerts %} for malware. For more information, see [AUTOTITLE](/code-security/security-advisories/working-with-global-security-advisories-from-the-github-advisory-database/about-the-github-advisory-database#malware-advisories). diff --git a/data/reusables/security-configurations/overview.md b/data/reusables/security-configurations/overview.md deleted file mode 100644 index 6af55e86f2e7..000000000000 --- a/data/reusables/security-configurations/overview.md +++ /dev/null @@ -1,7 +0,0 @@ -{% ifversion security-configurations-cloud %} - -We recommend securing your enterprise with the {% data variables.product.prodname_github_security_configuration %}, then evaluating the security findings on your repositories before configuring {% data variables.product.prodname_custom_security_configurations %}. For more information, see [AUTOTITLE](/admin/managing-code-security/securing-your-enterprise/applying-the-github-recommended-security-configuration-to-your-enterprise). - -{% endif %} - -With {% data variables.product.prodname_custom_security_configurations %}, you can create collections of enablement settings for {% data variables.product.company_short %}'s security products to meet the specific security needs of your enterprise. For example, you can create a different {% data variables.product.prodname_custom_security_configuration %} for each organization or group of similar organizations to reflect their different levels of security requirements and compliance obligations. For more information, see [AUTOTITLE](/admin/managing-code-security/securing-your-enterprise/creating-a-custom-security-configuration-for-your-enterprise). From 40c32fc116308cee5bc2fe6d22a5f0282ccedc49 Mon Sep 17 00:00:00 2001 From: Sam Browning <106113886+sabrowning1@users.noreply.github.com> Date: Thu, 12 Feb 2026 12:48:00 -0500 Subject: [PATCH 07/18] Re-add info on preserving default repo settings for security configs (#59604) Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com> Co-authored-by: Panagiotis Lithadiotis <55960073+panoslith@users.noreply.github.com> --- .../security-at-scale/security-configurations.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/content/code-security/concepts/security-at-scale/security-configurations.md b/content/code-security/concepts/security-at-scale/security-configurations.md index a07ec408ff2a..2a44b01d7eb6 100644 --- a/content/code-security/concepts/security-at-scale/security-configurations.md +++ b/content/code-security/concepts/security-at-scale/security-configurations.md @@ -87,6 +87,16 @@ Some situations can break the enforcement of {% data variables.product.prodname_ * Self-hosted runners with the label `code-scanning` are not available.{% endif %} * The languages excluded from {% data variables.product.prodname_code_scanning %} default setup are changed at the repository level. +{% ifversion security-configuration-enterprise-level %} + +## Preservation of default settings for new repositories + +If you had default security settings in place for newly created repositories, {% data variables.product.github %} will preserve these settings by automatically creating a "New repository default settings" {% data variables.product.prodname_security_configuration %} for your enterprise. The configuration matches your previous enterprise-level default settings for new repositories as of December 2024. + +The configuration will be automatically applied to any newly created repositories in your enterprise that do not belong to an organization with its own default settings. + +{% endif %} + ## Next steps {% ifversion security-configurations-cloud %} From d3b0beaeab59ed70e1dc032bba46dfe4b8452ffb Mon Sep 17 00:00:00 2001 From: Tim Rogers Date: Thu, 12 Feb 2026 20:50:02 +0300 Subject: [PATCH 08/18] Correct Auto models list for Copilot coding agent (#59514) Co-authored-by: Ben Ahmady <32935794+subatoi@users.noreply.github.com> --- data/reusables/copilot/copilot-coding-agent-auto-models.md | 3 +-- data/reusables/copilot/copilot-coding-agent-non-auto-models.md | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/reusables/copilot/copilot-coding-agent-auto-models.md b/data/reusables/copilot/copilot-coding-agent-auto-models.md index 21288f718ff9..19de1dfaf54c 100644 --- a/data/reusables/copilot/copilot-coding-agent-auto-models.md +++ b/data/reusables/copilot/copilot-coding-agent-auto-models.md @@ -1,2 +1 @@ -* {% data variables.copilot.copilot_claude_sonnet_45 %} -* {% data variables.copilot.copilot_gpt_51_codex_max %} \ No newline at end of file +* {% data variables.copilot.copilot_claude_sonnet_45 %} \ No newline at end of file diff --git a/data/reusables/copilot/copilot-coding-agent-non-auto-models.md b/data/reusables/copilot/copilot-coding-agent-non-auto-models.md index ecccd4824da5..4c7a642532a9 100644 --- a/data/reusables/copilot/copilot-coding-agent-non-auto-models.md +++ b/data/reusables/copilot/copilot-coding-agent-non-auto-models.md @@ -1,3 +1,4 @@ * {% data variables.copilot.copilot_claude_opus_45 %} * {% data variables.copilot.copilot_claude_opus_46 %} +* {% data variables.copilot.copilot_gpt_51_codex_max %} * {% data variables.copilot.copilot_gpt_52_codex %} From 74f661bcda7796540be58dd098a27ea5cad822fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:50:59 +0000 Subject: [PATCH 09/18] Bump axios from 1.12.1 to 1.13.5 in the npm_and_yarn group across 1 directory (#59605) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kevin Heis --- package-lock.json | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 520c5e5cb78a..24cffe909c4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2985,7 +2985,6 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.1.tgz", "integrity": "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -3025,7 +3024,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3046,7 +3044,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3067,7 +3064,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3088,7 +3084,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3109,7 +3104,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3130,7 +3124,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3151,7 +3144,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3172,7 +3164,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3193,7 +3184,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3214,7 +3204,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3235,7 +3224,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3256,7 +3244,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3277,7 +3264,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3295,7 +3281,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", - "dev": true, "license": "Apache-2.0", "optional": true, "bin": { @@ -5479,13 +5464,13 @@ } }, "node_modules/axios": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.1.tgz", - "integrity": "sha512-Kn4kbSXpkFHCGE6rBFNwIv0GQs4AvDT80jlveJDKFxjbTYMUeB4QtsdPCv6H8Cm19Je7IU6VFtRl2zWZI0rudQ==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.4", + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", "proxy-from-env": "^1.1.0" } }, @@ -8631,15 +8616,16 @@ "license": "W3C" }, "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", "funding": [ { "type": "individual", "url": "https://github.com/sponsors/RubenVerborgh" } ], + "license": "MIT", "engines": { "node": ">=4.0" }, @@ -8694,9 +8680,9 @@ } }, "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -8755,7 +8741,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -12726,7 +12711,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", - "dev": true, "license": "MIT", "optional": true }, From 343af816cd4ba923ba6ab1b7f4d5f7b5ad73d165 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:52:13 +0000 Subject: [PATCH 10/18] Bump glob from 11.1.0 to 13.0.2 (#59598) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kevin Heis --- package-lock.json | 233 ++++++++++++++-------------------------------- package.json | 2 +- 2 files changed, 73 insertions(+), 162 deletions(-) diff --git a/package-lock.json b/package-lock.json index 24cffe909c4f..bff3c9a13c19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,7 +45,7 @@ "file-type": "21.0.0", "flat": "^6.0.1", "github-slugger": "^2.0.0", - "glob": "11.1.0", + "glob": "13.0.2", "hast-util-from-parse5": "^8.0.3", "hast-util-to-string": "^3.0.1", "hastscript": "^9.0.1", @@ -2321,66 +2321,13 @@ } }, "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", + "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", + "dev": true, + "license": "BlueOak-1.0.0", "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + "node": ">=18" } }, "node_modules/@jridgewell/gen-mapping": { @@ -6468,6 +6415,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -6937,12 +6885,6 @@ "dev": true, "license": "MIT" }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "license": "MIT" - }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", @@ -6963,6 +6905,7 @@ }, "node_modules/emoji-regex": { "version": "9.2.2", + "dev": true, "license": "MIT" }, "node_modules/encodeurl": { @@ -8655,6 +8598,7 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, "license": "ISC", "dependencies": { "cross-spawn": "^7.0.6", @@ -8671,6 +8615,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, "license": "ISC", "engines": { "node": ">=14" @@ -8896,21 +8841,15 @@ "license": "ISC" }, "node_modules/glob": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", - "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.2.tgz", + "integrity": "sha512-035InabNu/c1lW0tzPhAgapKctblppqsKKG9ZaNzbr+gXwWMjXoiyGSyB9sArzrjG7jY+zntRq5ZSUYemrnWVQ==", "license": "BlueOak-1.0.0", "dependencies": { - "foreground-child": "^3.3.1", - "jackspeak": "^4.1.1", - "minimatch": "^10.1.1", + "minimatch": "^10.1.2", "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", "path-scurry": "^2.0.0" }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, "engines": { "node": "20 || >=22" }, @@ -8930,12 +8869,12 @@ } }, "node_modules/glob/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.2.tgz", + "integrity": "sha512-fu656aJ0n2kcXwsnwnv9g24tkU5uSmOlTjd6WyyaKm2Z+h1qmY6bAjrcaIxF/BslFqbZ8UBtbJi7KgQOZD2PTw==", "license": "BlueOak-1.0.0", "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" + "@isaacs/brace-expansion": "^5.0.1" }, "engines": { "node": "20 || >=22" @@ -10349,15 +10288,17 @@ }, "node_modules/isexe": { "version": "2.0.0", + "dev": true, "license": "ISC" }, "node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", + "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", + "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "@isaacs/cliui": "^8.0.2" + "@isaacs/cliui": "^9.0.0" }, "engines": { "node": "20 || >=22" @@ -13158,7 +13099,8 @@ "node_modules/package-json-from-dist": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", - "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==" + "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==", + "dev": true }, "node_modules/parent-module": { "version": "1.0.1", @@ -13346,6 +13288,7 @@ }, "node_modules/path-key": { "version": "3.1.1", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -14184,6 +14127,47 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rimraf/node_modules/glob": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", + "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/minimatch": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.2.tgz", + "integrity": "sha512-fu656aJ0n2kcXwsnwnv9g24tkU5uSmOlTjd6WyyaKm2Z+h1qmY6bAjrcaIxF/BslFqbZ8UBtbJi7KgQOZD2PTw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.1" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/robots-parser": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", @@ -14660,6 +14644,7 @@ }, "node_modules/shebang-command": { "version": "2.0.0", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -14670,6 +14655,7 @@ }, "node_modules/shebang-regex": { "version": "3.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -15082,39 +15068,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/string-width/node_modules/emoji-regex": { "version": "8.0.0", "license": "MIT" @@ -15229,19 +15182,6 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi/node_modules/ansi-regex": { "version": "6.0.1", "license": "MIT", @@ -16840,6 +16780,7 @@ }, "node_modules/which": { "version": "2.0.2", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -17016,36 +16957,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/wrap-ansi/node_modules/ansi-styles": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", diff --git a/package.json b/package.json index a612cf398cb7..beae8b9e4489 100644 --- a/package.json +++ b/package.json @@ -195,7 +195,7 @@ "file-type": "21.0.0", "flat": "^6.0.1", "github-slugger": "^2.0.0", - "glob": "11.1.0", + "glob": "13.0.2", "hast-util-from-parse5": "^8.0.3", "hast-util-to-string": "^3.0.1", "hastscript": "^9.0.1", From 28ce1043b480482eb111a34423f7a198a0eb9432 Mon Sep 17 00:00:00 2001 From: docs-bot <77750099+docs-bot@users.noreply.github.com> Date: Thu, 12 Feb 2026 09:53:37 -0800 Subject: [PATCH 11/18] Update audit log event data (#59602) Co-authored-by: Kevin Heis --- src/audit-logs/data/fpt/organization.json | 34 ++++++++++++++++++++++ src/audit-logs/data/ghec/enterprise.json | 34 ++++++++++++++++++++++ src/audit-logs/data/ghec/organization.json | 34 ++++++++++++++++++++++ src/audit-logs/lib/config.json | 2 +- 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/audit-logs/data/fpt/organization.json b/src/audit-logs/data/fpt/organization.json index f08e99a42520..a2624a1c21ba 100644 --- a/src/audit-logs/data/fpt/organization.json +++ b/src/audit-logs/data/fpt/organization.json @@ -21185,6 +21185,40 @@ ], "docs_reference_titles": "Enabling extended metadata checks for your repository" }, + { + "action": "repository_secret_scanning_extended_metadata.enabled", + "description": "Metadata for secret scanning alerts has been enabled at the repository level", + "docs_reference_links": "/code-security/secret-scanning/enabling-secret-scanning-features/enabling-extended-metadata-checks-for-your-repository", + "fields": [ + "@timestamp", + "_document_id", + "action", + "actor", + "actor_id", + "business", + "business_id", + "hashed_token", + "org", + "org_id", + "programmatic_access_type", + "repo", + "repo_id", + "repository", + "repository_id", + "request_access_security_header", + "request_id", + "token_id", + "token_scopes", + "user", + "user_id", + "user_agent", + "public_repo", + "created_at", + "operation_type", + "actor_is_bot" + ], + "docs_reference_titles": "Enabling extended metadata checks for your repository" + }, { "action": "repository_secret_scanning_generic_secrets.disabled", "description": "Generic secrets have been disabled at the repository level", diff --git a/src/audit-logs/data/ghec/enterprise.json b/src/audit-logs/data/ghec/enterprise.json index 2c4795b8b6ca..f536162bc9a1 100644 --- a/src/audit-logs/data/ghec/enterprise.json +++ b/src/audit-logs/data/ghec/enterprise.json @@ -26564,6 +26564,40 @@ ], "docs_reference_titles": "Enabling extended metadata checks for your repository" }, + { + "action": "repository_secret_scanning_extended_metadata.enabled", + "description": "Metadata for secret scanning alerts has been enabled at the repository level", + "docs_reference_links": "/code-security/secret-scanning/enabling-secret-scanning-features/enabling-extended-metadata-checks-for-your-repository", + "fields": [ + "@timestamp", + "_document_id", + "action", + "actor", + "actor_id", + "business", + "business_id", + "hashed_token", + "org", + "org_id", + "programmatic_access_type", + "repo", + "repo_id", + "repository", + "repository_id", + "request_access_security_header", + "request_id", + "token_id", + "token_scopes", + "user", + "user_id", + "user_agent", + "public_repo", + "created_at", + "operation_type", + "actor_is_bot" + ], + "docs_reference_titles": "Enabling extended metadata checks for your repository" + }, { "action": "repository_secret_scanning_generic_secrets.disabled", "description": "Generic secrets have been disabled at the repository level", diff --git a/src/audit-logs/data/ghec/organization.json b/src/audit-logs/data/ghec/organization.json index f08e99a42520..a2624a1c21ba 100644 --- a/src/audit-logs/data/ghec/organization.json +++ b/src/audit-logs/data/ghec/organization.json @@ -21185,6 +21185,40 @@ ], "docs_reference_titles": "Enabling extended metadata checks for your repository" }, + { + "action": "repository_secret_scanning_extended_metadata.enabled", + "description": "Metadata for secret scanning alerts has been enabled at the repository level", + "docs_reference_links": "/code-security/secret-scanning/enabling-secret-scanning-features/enabling-extended-metadata-checks-for-your-repository", + "fields": [ + "@timestamp", + "_document_id", + "action", + "actor", + "actor_id", + "business", + "business_id", + "hashed_token", + "org", + "org_id", + "programmatic_access_type", + "repo", + "repo_id", + "repository", + "repository_id", + "request_access_security_header", + "request_id", + "token_id", + "token_scopes", + "user", + "user_id", + "user_agent", + "public_repo", + "created_at", + "operation_type", + "actor_is_bot" + ], + "docs_reference_titles": "Enabling extended metadata checks for your repository" + }, { "action": "repository_secret_scanning_generic_secrets.disabled", "description": "Generic secrets have been disabled at the repository level", diff --git a/src/audit-logs/lib/config.json b/src/audit-logs/lib/config.json index 09fb4eddfa22..1d889e99bd88 100644 --- a/src/audit-logs/lib/config.json +++ b/src/audit-logs/lib/config.json @@ -9,5 +9,5 @@ "git": "Note: Git events have special access requirements and retention policies that differ from other audit log events. For GitHub Enterprise Cloud, access Git events via the REST API only with 7-day retention. For GitHub Enterprise Server, Git events must be enabled in audit log configuration and are not included in search results.", "sso_redirect": "Note: Automatically redirecting users to sign in is currently in beta for Enterprise Managed Users and subject to change." }, - "sha": "b426a3c9af4b379c4fdd7ee0d59c9db7edb11565" + "sha": "6fbd793fd982298d46c7481b7d66f7f3623fb5f0" } \ No newline at end of file From 3b6fd2f1a872f9f88bb0c4ac9e952050afb2c562 Mon Sep 17 00:00:00 2001 From: mc <42146119+mchammer01@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:56:17 +0000 Subject: [PATCH 12/18] [EDI] Configuring private vulnerability reporting for a repository (#59584) --- ...inated-disclosure-of-security-vulnerabilities.md | 13 +++++++++---- ...vate-vulnerability-reporting-for-a-repository.md | 9 +-------- .../private-vulnerability-reporting-benefits.md | 10 ++++------ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/content/code-security/concepts/vulnerability-reporting-and-management/about-coordinated-disclosure-of-security-vulnerabilities.md b/content/code-security/concepts/vulnerability-reporting-and-management/about-coordinated-disclosure-of-security-vulnerabilities.md index 0643dc709ff4..f00d5106e530 100644 --- a/content/code-security/concepts/vulnerability-reporting-and-management/about-coordinated-disclosure-of-security-vulnerabilities.md +++ b/content/code-security/concepts/vulnerability-reporting-and-management/about-coordinated-disclosure-of-security-vulnerabilities.md @@ -84,11 +84,16 @@ Private vulnerability reporting provides a secure, structured way for security r Without clear guidance on how to contact maintainers, security researchers may feel forced to disclose vulnerabilities publicly, such as by posting on social media, opening public issues, or contacting maintainers through informal channels, which can expose users to unnecessary risk. Private vulnerability reporting helps avoid these situations by offering a dedicated, private reporting workflow. -For security researchers, private vulnerability reporting offers: +For security researchers, the benefits of using private vulnerability reporting are: -* Less frustration, and less time spent trying to figure out how to contact the maintainer. -* A smoother process for disclosing and discussing vulnerability details. -* The opportunity to discuss vulnerability details privately with the repository maintainer. +* A clear, structured way to contact maintainers +* A smoother process for disclosing and discussing vulnerability details +* The ability to discuss vulnerability details privately with the repository maintainer +* Reduced risk of vulnerability details being in the public eye before a fix is available + +For maintainers, the benefits of using private vulnerability reporting are: + +{% data reusables.security-advisory.private-vulnerability-reporting-benefits %} For more information for security researchers and repository maintainers, see [AUTOTITLE](/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) and [AUTOTITLE](/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/managing-privately-reported-security-vulnerabilities), respectively. diff --git a/content/code-security/how-tos/report-and-fix-vulnerabilities/configure-vulnerability-reporting/configuring-private-vulnerability-reporting-for-a-repository.md b/content/code-security/how-tos/report-and-fix-vulnerabilities/configure-vulnerability-reporting/configuring-private-vulnerability-reporting-for-a-repository.md index 8f72abc1a0a2..9907db246196 100644 --- a/content/code-security/how-tos/report-and-fix-vulnerabilities/configure-vulnerability-reporting/configuring-private-vulnerability-reporting-for-a-repository.md +++ b/content/code-security/how-tos/report-and-fix-vulnerabilities/configure-vulnerability-reporting/configuring-private-vulnerability-reporting-for-a-repository.md @@ -15,14 +15,7 @@ redirect_from: - /code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository --- -## About privately reporting a security vulnerability - -Security researchers often feel responsible for alerting users to a vulnerability that could be exploited. If there are no clear instructions about contacting maintainers of the repository containing the vulnerability, security researchers may have no other choice but to post about the vulnerability on social media, send direct messages to the maintainer, or even create public issues. This situation can potentially lead to a public disclosure of the vulnerability details. - -{% data reusables.security-advisory.private-vulnerability-reporting-overview %} - -For maintainers, the benefits of using private vulnerability reporting are: -{% data reusables.security-advisory.private-vulnerability-reporting-benefits %} +Enabling private vulnerability reporting gives security researchers a secure, structured way to disclose vulnerabilities directly in your repository. Once enabled, researchers can submit reports through without resorting to public disclosure or informal channels. For background on private vulnerability reporting and how it fits into coordinated disclosure, see [AUTOTITLE](/code-security/concepts/vulnerability-reporting-and-management/about-coordinated-disclosure-of-security-vulnerabilities). The instructions in this article refer to enablement at repository level. For information about enabling the feature at organization level, see [AUTOTITLE](/code-security/security-advisories/working-with-repository-security-advisories/configuring-private-vulnerability-reporting-for-an-organization). diff --git a/data/reusables/security-advisory/private-vulnerability-reporting-benefits.md b/data/reusables/security-advisory/private-vulnerability-reporting-benefits.md index d446e9964797..96c014cb220d 100644 --- a/data/reusables/security-advisory/private-vulnerability-reporting-benefits.md +++ b/data/reusables/security-advisory/private-vulnerability-reporting-benefits.md @@ -1,6 +1,4 @@ -* Less risk of being contacted publicly, or via undesired means. -* Receive reports in the same platform you resolve them in for simplicity -* The security researcher creates or at least initiates the advisory report on the behalf of maintainers. -* Maintainers receive reports in the same platform as the one used to discuss and resolve the advisories. -* Vulnerability less likely to be in the public eye. -* The opportunity to discuss vulnerability details privately with security researchers and collaborate on the patch. +* Receiving reports in the same platform where they are resolved +* Security researchers creating or initiating the advisory report on behalf of maintainers +* Reduced risk of vulnerabilities being in the public eye before a fix is available +* The opportunity to discuss vulnerability details privately with security researchers and collaborate on the patch From f1d891c561fb4bb91b24cd7551a3d19c813e41a1 Mon Sep 17 00:00:00 2001 From: Erik Bershel <110455084+erik-bershel@users.noreply.github.com> Date: Thu, 12 Feb 2026 18:56:42 +0100 Subject: [PATCH 13/18] macOS-13, Ubuntu-20.04 were deprecated; Ubuntu-slim runs in unprivileged mode (#59590) --- .../how-tos/manage-runners/larger-runners/control-access.md | 2 +- .../manage-runners/larger-runners/use-larger-runners.md | 6 +++--- content/actions/reference/runners/github-hosted-runners.md | 3 +++ .../reference/workflows-and-actions/workflow-syntax.md | 2 +- data/reusables/actions/add-hosted-runner.md | 2 +- .../actions/jobs/example-runs-on-labels-and-groups.md | 4 ++-- data/reusables/actions/larger-runners-table.md | 4 ++-- data/reusables/actions/macos-runner-limitations.md | 2 +- data/reusables/actions/supported-github-runners.md | 2 -- 9 files changed, 14 insertions(+), 13 deletions(-) diff --git a/content/actions/how-tos/manage-runners/larger-runners/control-access.md b/content/actions/how-tos/manage-runners/larger-runners/control-access.md index 1eae8b439b89..e61536163238 100644 --- a/content/actions/how-tos/manage-runners/larger-runners/control-access.md +++ b/content/actions/how-tos/manage-runners/larger-runners/control-access.md @@ -27,7 +27,7 @@ Runner groups are used to control which repositories can run jobs on your {% dat * **Runners at the enterprise level:** {% data reusables.actions.about-enterprise-level-runner-groups %} * **Runners at the organization level:** {% data reusables.actions.about-organization-level-runner-groups %} -For example, the following diagram has a runner group named `grp-ubuntu-20.04-16core` at the enterprise level. Before the repository named `octo-repo` can use the runners in the group, you must first configure the group at the enterprise level to allow access to the `octo-org` organization. You must then configure the group at the organization level to allow access to `octo-repo`. +For example, the following diagram has a runner group named `grp-ubuntu-24.04-16core` at the enterprise level. Before the repository named `octo-repo` can use the runners in the group, you must first configure the group at the enterprise level to allow access to the `octo-org` organization. You must then configure the group at the organization level to allow access to `octo-repo`. ![Diagram showing a runner group defined at the enterprise level with an organization configuration that allows access for two repositories.](/assets/images/help/actions/hosted-runner-mgmt.png) diff --git a/content/actions/how-tos/manage-runners/larger-runners/use-larger-runners.md b/content/actions/how-tos/manage-runners/larger-runners/use-larger-runners.md index 53a01c891c73..43b2945db7a1 100644 --- a/content/actions/how-tos/manage-runners/larger-runners/use-larger-runners.md +++ b/content/actions/how-tos/manage-runners/larger-runners/use-larger-runners.md @@ -85,7 +85,7 @@ Use the labels in the table below to run your workflows on the corresponding mac {% data reusables.actions.runner-labels-implicit %} -In this example, the `runs-on` key sends the job to any available runner that has been assigned the `ubuntu-20.04-16core` label: +In this example, the `runs-on` key sends the job to any available runner that has been assigned the `ubuntu-24.04-16core` label: ```yaml name: learn-github-actions @@ -93,7 +93,7 @@ on: [push] jobs: check-bats-version: runs-on: - labels: ubuntu-20.04-16core + labels: ubuntu-24.04-16core steps: - uses: {% data reusables.actions.action-checkout %} - uses: {% data reusables.actions.action-setup-node %} @@ -148,7 +148,7 @@ name: learn-github-actions-testing on: [push] jobs: build: - runs-on: macos-13-xlarge + runs-on: macos-26-xlarge steps: - uses: {% data reusables.actions.action-checkout %} - name: Build diff --git a/content/actions/reference/runners/github-hosted-runners.md b/content/actions/reference/runners/github-hosted-runners.md index 6d57bb3e45bf..f7ad145edda9 100644 --- a/content/actions/reference/runners/github-hosted-runners.md +++ b/content/actions/reference/runners/github-hosted-runners.md @@ -50,6 +50,9 @@ Single-CPU {% data variables.product.github %}-hosted runners are available in b `ubuntu-slim` runners execute Actions workflows in Ubuntu Linux, inside a container rather than a full VM instance. When the job begins, {% data variables.product.github %} automatically provisions a new container for that job. All steps in the job execute in the container, allowing the steps in that job to share information using the runner's file system. When the job has finished, the container is automatically decommissioned. Each container provides hypervisor level 2 isolation. +> [!NOTE] +> The container for `ubuntu-slim` runners runs in unprivileged mode. This means that some operations requiring elevated privileges—such as mounting file systems, using Docker-in-Docker, or accessing low-level kernel features—are not supported. + A minimal set of tools is installed on the `ubuntu-slim` runner image, appropriate for lightweight tasks. For details on what software is installed on the `ubuntu-slim` image, see the [README file](https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md) in the `actions/runner-images` repository. #### Usage limits diff --git a/content/actions/reference/workflows-and-actions/workflow-syntax.md b/content/actions/reference/workflows-and-actions/workflow-syntax.md index c00653b7ec44..b916b5c66a4e 100644 --- a/content/actions/reference/workflows-and-actions/workflow-syntax.md +++ b/content/actions/reference/workflows-and-actions/workflow-syntax.md @@ -953,7 +953,7 @@ jobs: example_matrix: strategy: matrix: - os: [ubuntu-22.04, ubuntu-20.04] + os: [ubuntu-22.04, ubuntu-24.04] version: [10, 12, 14] runs-on: {% raw %}${{ matrix.os }}{% endraw %} steps: diff --git a/data/reusables/actions/add-hosted-runner.md b/data/reusables/actions/add-hosted-runner.md index 4ad6e7d8e3f5..4b319a7ef933 100644 --- a/data/reusables/actions/add-hosted-runner.md +++ b/data/reusables/actions/add-hosted-runner.md @@ -1,7 +1,7 @@ 1. Click **New runner**, then click **{% octicon "mark-github" aria-hidden="true" aria-label="mark-github" %} New {% data variables.product.prodname_dotcom %}-hosted runner**. 1. Complete the required details to configure your new runner: - * **Name:** Enter a name for your new runner. For easier identification, this should indicate its hardware and operating configuration, such as `ubuntu-20.04-16core`. + * **Name:** Enter a name for your new runner. For easier identification, this should indicate its hardware and operating configuration, such as `ubuntu-24.04-16core`. * **Platform:** Choose a platform from the available options. Once you've selected a platform, you will be able to choose a specific image. If you are building a custom image, the platform that you select for your runner must match the platform of the image you want to build. The platform of the runner can be one of the following: diff --git a/data/reusables/actions/jobs/example-runs-on-labels-and-groups.md b/data/reusables/actions/jobs/example-runs-on-labels-and-groups.md index 17a4cb297461..fae7f2d9dd75 100644 --- a/data/reusables/actions/jobs/example-runs-on-labels-and-groups.md +++ b/data/reusables/actions/jobs/example-runs-on-labels-and-groups.md @@ -1,6 +1,6 @@ When you combine groups and labels, the runner must meet both requirements to be eligible to run the job. -In this example, a runner group called `ubuntu-runners` is populated with Ubuntu runners, which have also been assigned the label `ubuntu-20.04-16core`. The `runs-on` key combines `group` and `labels` so that the job is routed to any available runner within the group that also has a matching label: +In this example, a runner group called `ubuntu-runners` is populated with Ubuntu runners, which have also been assigned the label `ubuntu-24.04-16core`. The `runs-on` key combines `group` and `labels` so that the job is routed to any available runner within the group that also has a matching label: ```yaml name: learn-github-actions @@ -9,7 +9,7 @@ jobs: check-bats-version: runs-on: group: ubuntu-runners - labels: ubuntu-20.04-16core + labels: ubuntu-24.04-16core steps: - uses: {% data reusables.actions.action-checkout %} - uses: {% data reusables.actions.action-setup-node %} diff --git a/data/reusables/actions/larger-runners-table.md b/data/reusables/actions/larger-runners-table.md index 60c918ca12c0..ae2c35d3a70d 100644 --- a/data/reusables/actions/larger-runners-table.md +++ b/data/reusables/actions/larger-runners-table.md @@ -1,4 +1,4 @@ | Runner Size | Architecture| Processor (CPU)| Memory (RAM) | Storage (SSD) | Workflow label | | ------------| ------------| -------------- | ------------- | ------------- |--------------------------------------------------------------------------------------------------------------------------------------------------| -| Large | Intel | 12 | 30 GB | 14 GB | macos-latest-large, macos-13-large, macos-14-large, macos-15-large (latest), macos-26-large ({% data variables.release-phases.public_preview %}) | -| XLarge | arm64 (M2) | 5 (+ 8 GPU hardware acceleration) | 14 GB | 14 GB | macos-latest-xlarge, macos-13-xlarge, macos-14-xlarge, macos-15-xlarge (latest), macos-26-xlarge ({% data variables.release-phases.public_preview %}) | +| Large | Intel | 12 | 30 GB | 14 GB | macos-latest-large, macos-14-large, macos-15-large (latest), macos-26-large ({% data variables.release-phases.public_preview %}) | +| XLarge | arm64 (M2) | 5 (+ 8 GPU hardware acceleration) | 14 GB | 14 GB | macos-latest-xlarge, macos-14-xlarge, macos-15-xlarge (latest), macos-26-xlarge ({% data variables.release-phases.public_preview %}) | diff --git a/data/reusables/actions/macos-runner-limitations.md b/data/reusables/actions/macos-runner-limitations.md index 2672d25dc804..5dcd752feb07 100644 --- a/data/reusables/actions/macos-runner-limitations.md +++ b/data/reusables/actions/macos-runner-limitations.md @@ -1,5 +1,5 @@ * All actions provided by {% data variables.product.prodname_dotcom %} are compatible with arm64 {% data variables.product.prodname_dotcom %}-hosted runners. However, community actions may not be compatible with arm64 and need to be manually installed at runtime. -* Nested-virtualization and Metal Performance Shaders (MPS) are not supported due to the limitation of Apple's Virtualization Framework. +* Nested-virtualization is not supported due to the limitation of Apple's Virtualization Framework. * Networking capabilities such as Azure private networking and assigning static IPs are not currently available for macOS larger runners. * The arm64 macOS runners do not have a static UUID/UDID assigned to them because Apple does not support this feature. However, Intel MacOS runners are assigned a static UDID, specifically `4203018E-580F-C1B5-9525-B745CECA79EB`. If you are building and signing on the same host you plan to test the build on, you can sign with a [development provisioning profile](https://developer.apple.com/help/account/provisioning-profiles/create-a-development-provisioning-profile/). If you do require a static UDID, you can use Intel runners and add their UDID to your Apple Developer account. {%- ifversion ghec %} diff --git a/data/reusables/actions/supported-github-runners.md b/data/reusables/actions/supported-github-runners.md index c29a829dd9d6..974a815e79ef 100644 --- a/data/reusables/actions/supported-github-runners.md +++ b/data/reusables/actions/supported-github-runners.md @@ -68,7 +68,6 @@ For public repositories, jobs using the workflow labels shown in the table below 14 GB Intel - macos-13, macos-15-intel @@ -158,7 +157,6 @@ For {% ifversion ghec %}internal and{% endif %} private repositories, jobs using 14 GB Intel - macos-13, macos-15-intel From eac7bec3dbf3e38fa67b8452038cee0d06a51e84 Mon Sep 17 00:00:00 2001 From: Sophie <29382425+sophietheking@users.noreply.github.com> Date: Thu, 12 Feb 2026 18:56:48 +0100 Subject: [PATCH 14/18] Address survey comment on "Data available in Copilot usage metrics" (#59580) --- .../copilot-usage-metrics/copilot-usage-metrics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/content/copilot/reference/copilot-usage-metrics/copilot-usage-metrics.md b/content/copilot/reference/copilot-usage-metrics/copilot-usage-metrics.md index b4118ec1fe03..76140aa1ee64 100644 --- a/content/copilot/reference/copilot-usage-metrics/copilot-usage-metrics.md +++ b/content/copilot/reference/copilot-usage-metrics/copilot-usage-metrics.md @@ -19,10 +19,10 @@ category: {% data reusables.copilot.usage-metrics-preview %} -The {% data variables.product.prodname_copilot_short %} usage metrics dashboard and APIs display and export data using a consistent set of fields. This reference lists all available metrics and describes how to interpret their values in both dashboard visuals and NDJSON or API exports. To retrieve this data programmatically, see [AUTOTITLE](/rest/copilot/copilot-usage-metrics). +The {% data variables.product.prodname_copilot_short %} usage metrics dashboard and APIs display and export data using a consistent set of fields. This reference lists all available metrics and describes how to interpret their values in both dashboard visuals and NDJSON or API exports. -* The {% data variables.product.prodname_copilot_short %} usage metrics dashboard reports data at the **enterprise** level. -* The {% data variables.product.prodname_copilot_short %} usage metrics APIs support **enterprise-, organization-, and user-level** records. +* The {% data variables.product.prodname_copilot_short %} usage metrics dashboard reports data at the **enterprise** level. To access the dashboard, see [AUTOTITLE](/copilot/how-tos/administer-copilot/manage-for-enterprise/view-usage-and-adoption). +* The {% data variables.product.prodname_copilot_short %} usage metrics APIs support **enterprise-, organization-, and user-level** records. To retrieve this data programmatically, see [AUTOTITLE](/rest/copilot/copilot-usage-metrics). For guidance on how to read and interpret these metrics, see [AUTOTITLE](/copilot/concepts/copilot-metrics). From 79f9bf193356196d0c172bdc572ee326195b3c88 Mon Sep 17 00:00:00 2001 From: Brendan Scott-Smith <117171930+bss-mc@users.noreply.github.com> Date: Fri, 13 Feb 2026 02:56:56 +0900 Subject: [PATCH 15/18] Add that removing organizations is not possible with EMU (#59579) Co-authored-by: Justin Alex <1155821+jusuchin85@users.noreply.github.com> --- .../removing-organizations-from-your-enterprise.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/content/admin/managing-accounts-and-repositories/managing-organizations-in-your-enterprise/removing-organizations-from-your-enterprise.md b/content/admin/managing-accounts-and-repositories/managing-organizations-in-your-enterprise/removing-organizations-from-your-enterprise.md index 1282bb13413c..50bbe33e876b 100644 --- a/content/admin/managing-accounts-and-repositories/managing-organizations-in-your-enterprise/removing-organizations-from-your-enterprise.md +++ b/content/admin/managing-accounts-and-repositories/managing-organizations-in-your-enterprise/removing-organizations-from-your-enterprise.md @@ -14,6 +14,12 @@ redirect_from: You can remove an organization that is owned by your enterprise account, so the organization stands alone. +## Limitations + +If you use {% data variables.product.prodname_emus %} or {% data variables.enterprise.data_residency %}, removing organizations from your enterprise is not possible. + +If you use {% data variables.product.prodname_emus %}, you can instead migrate organizations with the {% data variables.product.prodname_importer_proper_name %}. See [AUTOTITLE](/migrations/using-github-enterprise-importer/migrating-between-github-products/about-migrations-between-github-products). + ## What happens when an organization is removed? When you remove an organization from your enterprise: From 5c9cdff8a4ea0943f6662564394e90545fbc7ece Mon Sep 17 00:00:00 2001 From: Jonathan Celestin <25135255+optimisticjc@users.noreply.github.com> Date: Thu, 12 Feb 2026 09:57:29 -0800 Subject: [PATCH 16/18] Update support status for Visual Studio in documentation for MCP Registry (#59583) --- content/copilot/concepts/mcp-management.md | 2 +- .../manage-mcp-usage/configure-mcp-registry.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/content/copilot/concepts/mcp-management.md b/content/copilot/concepts/mcp-management.md index a55de3b6265d..4097a7ae1a2f 100644 --- a/content/copilot/concepts/mcp-management.md +++ b/content/copilot/concepts/mcp-management.md @@ -41,7 +41,7 @@ MCP management features are supported as follows: | {% data variables.copilot.copilot_coding_agent %} | {% octicon "x" aria-label="Not supported" %} | {% octicon "x" aria-label="Not supported" %} | | Eclipse | {% octicon "check" aria-label="Supported" %} | {% octicon "check" aria-label="Supported" %} | | JetBrains | {% octicon "check" aria-label="Supported" %} | {% octicon "check" aria-label="Supported" %} | -| {% data variables.product.prodname_vs %} | {% octicon "x" aria-label="Not supported" %} | {% octicon "x" aria-label="Not supported" %} | +| {% data variables.product.prodname_vs %} | {% octicon "check" aria-label="Supported" %} | {% octicon "check" aria-label="Supported" %} | | {% data variables.product.prodname_vscode_shortname %} | {% octicon "check" aria-label="Supported" %} | {% octicon "check" aria-label="Supported" %} | | {% data variables.product.prodname_vscode_shortname %} Insiders | {% octicon "check" aria-label="Supported" %} | {% octicon "check" aria-label="Supported" %} | | Xcode | {% octicon "check" aria-label="Supported" %} | {% octicon "check" aria-label="Supported" %} | diff --git a/content/copilot/how-tos/administer-copilot/manage-mcp-usage/configure-mcp-registry.md b/content/copilot/how-tos/administer-copilot/manage-mcp-usage/configure-mcp-registry.md index 7602ec4f652e..a608f664e577 100644 --- a/content/copilot/how-tos/administer-copilot/manage-mcp-usage/configure-mcp-registry.md +++ b/content/copilot/how-tos/administer-copilot/manage-mcp-usage/configure-mcp-registry.md @@ -51,9 +51,9 @@ While the MCP registry launched using the v0 specification, that version is now | {% data variables.product.prodname_vscode_shortname %} Insiders | {% octicon "check" aria-label="Supported" %} | | {% data variables.product.prodname_vscode_shortname %} | {% octicon "check" aria-label="Supported" %} | | {% data variables.product.prodname_vs %} | {% octicon "check" aria-label="Supported" %} | -| Eclipse | Coming Dec 2025 | -| JetBrains IDEs | Coming Dec 2025 | -| Xcode | Coming Dec 2025 | +| Eclipse | {% octicon "check" aria-label="Supported" %} | +| JetBrains IDEs | {% octicon "check" aria-label="Supported" %} | +| Xcode | {% octicon "check" aria-label="Supported" %} | ### Cross-Origin Resource Sharing requirements From f1ec15ae06cda9a19ac82458314e8cb288f9f98f Mon Sep 17 00:00:00 2001 From: John Clement <70238417+jclement136@users.noreply.github.com> Date: Thu, 12 Feb 2026 12:57:36 -0500 Subject: [PATCH 17/18] Update readability agent file - add line on active verbs (#59578) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/agents/readability-editor.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/agents/readability-editor.md b/.github/agents/readability-editor.md index 082e72850057..1cfcf9159bc5 100644 --- a/.github/agents/readability-editor.md +++ b/.github/agents/readability-editor.md @@ -36,6 +36,7 @@ You are an expert editor for the GitHub Docs content team. Your job is to maximi - When two possible phrasings are equally clear, choose the one with fewer words. Brevity directly improves readability. - Use full terms and not their shortened versions. - Use active voice and personal pronouns ("you," "your"); favor present tense. +- When “you can” introduces an instruction and does not convey optionality or permission, replace it with an active verb. For example, “You can enable” becomes “Enable”. Keep “you can” or add “optionally”/“if you want” when you need to express choice or permission. - Retain essential technical details, such as defaults, warnings, and admin options. - Do not alter the intent of verbs and actions (ex. "navigate" does not necessarily mean "select"). - Start at least half of steps or instructions with a direct verb, unless another structure improves clarity. From 3b1bff97a331485741f38320032d8fba75de7432 Mon Sep 17 00:00:00 2001 From: Sam Browning <106113886+sabrowning1@users.noreply.github.com> Date: Thu, 12 Feb 2026 12:57:43 -0500 Subject: [PATCH 18/18] [EDI] Editing security advisories in the GitHub Advisory Database (#59577) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com> --- .../about-the-github-advisory-database.md | 17 ++++++++++++----- ...dvisories-in-the-github-advisory-database.md | 14 ++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/content/code-security/concepts/vulnerability-reporting-and-management/about-the-github-advisory-database.md b/content/code-security/concepts/vulnerability-reporting-and-management/about-the-github-advisory-database.md index 24cf0c5fdd59..ee00f8dcd5b5 100644 --- a/content/code-security/concepts/vulnerability-reporting-and-management/about-the-github-advisory-database.md +++ b/content/code-security/concepts/vulnerability-reporting-and-management/about-the-github-advisory-database.md @@ -23,7 +23,7 @@ redirect_from: Security advisories are published as JSON files in the Open Source Vulnerability (OSV) format. For more information about the OSV format, see [Open Source Vulnerability format](https://ossf.github.io/osv-schema/). -## About types of security advisories +## Types of security advisories Each advisory in the {% data variables.product.prodname_advisory_database %} is for a vulnerability in open source projects or for malicious open source software. @@ -68,11 +68,11 @@ If you enable {% data variables.product.prodname_dependabot_alerts %} for your r Our malware advisories are mostly about substitution attacks. During this type of attack, an attacker publishes a package to the public registry with the same name as a dependency that users rely on from a third party or private registry, with the hope that the malicious version is consumed. {% data variables.product.prodname_dependabot %} doesn’t look at project configurations to determine if the packages are coming from a private registry, so we aren't sure if you're using the malicious version or a non-malicious version. Users who have their dependencies appropriately scoped should not be affected by malware. -## About information in security advisories +## Information in security advisories In this section, you can find more detailed information about specific data attributes of the {% data variables.product.prodname_advisory_database %}. -### About GHSA IDs +### GHSA IDs Each security advisory, regardless of its type, has a unique identifier referred to as a GHSA ID. A `GHSA-ID` qualifier is assigned when a new advisory is created on {% data variables.product.prodname_dotcom %} or added to the {% data variables.product.prodname_advisory_database %} from any of the supported sources. @@ -89,7 +89,7 @@ You can validate a GHSA ID using a regular expression. /GHSA(-[23456789cfghjmpqrvwx]{4}){3}/ ``` -### About CVSS levels +### CVSS levels {% ifversion cvss-4 %} The {% data variables.product.prodname_advisory_database %} supports both CVSS version 3.1 and CVSS version 4.0.{% endif %} @@ -105,7 +105,7 @@ The {% data variables.product.prodname_advisory_database %} uses the CVSS levels {% data reusables.repositories.github-security-lab %} -### About EPSS scores +### EPSS scores The Exploit Prediction Scoring System, or EPSS, is a system devised by the global Forum of Incident Response and Security Teams (FIRST) for quantifying the likelihood of vulnerability exploit. The model produces a probability score between 0 and 1 (0 and 100%), where the higher the score, the greater the probability that a vulnerability will be exploited. For more information about FIRST, see https://www.first.org/. @@ -124,6 +124,13 @@ FIRST also provides additional information around the distribution of their EPSS At {% data variables.product.company_short %}, we do not author this data, but rather source it from FIRST, which means that this data is not editable in community contributions. For more information about community contributions, see [AUTOTITLE](/code-security/security-advisories/working-with-global-security-advisories-from-the-github-advisory-database/editing-security-advisories-in-the-github-advisory-database). +## Community contributions + +A **community contribution** is a pull request submitted to the [`github/advisory-database`](https://github.com/github/advisory-database) repository that improves the content of a global security advisory. When you make a community contribution, you can edit or add any detail, including additional affected ecosystems, the severity level, or the description of who is impacted. The {% data variables.product.prodname_security %} curation team will review the submitted contributions and publish them onto the {% data variables.product.prodname_advisory_database %} if accepted. + +{% ifversion security-advisories-credit-types %} +If we accept and publish the community contribution, the person who submitted the community contribution pull request will automatically be assigned a credit type of "Analyst". For more information, see [AUTOTITLE](/code-security/security-advisories/working-with-repository-security-advisories/creating-a-repository-security-advisory#about-credits-for-repository-security-advisories).{% endif %} + ## Further reading * [AUTOTITLE](/code-security/dependabot/dependabot-alerts/about-dependabot-alerts) diff --git a/content/code-security/how-tos/report-and-fix-vulnerabilities/fix-reported-vulnerabilities/editing-security-advisories-in-the-github-advisory-database.md b/content/code-security/how-tos/report-and-fix-vulnerabilities/fix-reported-vulnerabilities/editing-security-advisories-in-the-github-advisory-database.md index 122e4575852f..bb2abb6f7273 100644 --- a/content/code-security/how-tos/report-and-fix-vulnerabilities/fix-reported-vulnerabilities/editing-security-advisories-in-the-github-advisory-database.md +++ b/content/code-security/how-tos/report-and-fix-vulnerabilities/fix-reported-vulnerabilities/editing-security-advisories-in-the-github-advisory-database.md @@ -1,6 +1,6 @@ --- title: Editing security advisories in the GitHub Advisory Database -intro: You can submit improvements to any advisory published in the {% data variables.product.prodname_advisory_database %} by making a community contribution. +intro: Improve advisories published in the {% data variables.product.prodname_advisory_database %} by making community contributions. permissions: '{% data reusables.permissions.global-security-advisories-edit %}' redirect_from: - /code-security/security-advisories/editing-security-advisories-in-the-github-advisory-database @@ -22,17 +22,11 @@ topics: shortTitle: Edit Advisory Database --- -## Editing advisories in the {% data variables.product.prodname_advisory_database %} - -The advisories in the {% data variables.product.prodname_advisory_database %} are global security advisories. For more information about global security advisories, see [AUTOTITLE](/code-security/security-advisories/working-with-global-security-advisories-from-the-github-advisory-database/about-global-security-advisories). - -Anyone can suggest improvements on any global security advisory in the {% data variables.product.prodname_advisory_database %} by making a **community contribution**. A **community contribution** is a pull request submitted to the [github/advisory-database](https://github.com/github/advisory-database) repository that improves the content of a global security advisory. When you make a community contribution, you can edit or add any detail, including additionally affected ecosystems, severity level or description of who is impacted. The {% data variables.product.prodname_security %} curation team will review the submitted contributions and publish them onto the {% data variables.product.prodname_advisory_database %} if accepted. +{% ifversion ghes %} -{% ifversion security-advisories-credit-types %} -If we accept and publish the community contribution, the person who submitted the community contribution pull request will automatically be assigned a credit type of "Analyst". For more information, see [AUTOTITLE](/code-security/security-advisories/working-with-repository-security-advisories/creating-a-repository-security-advisory#about-credits-for-repository-security-advisories).{% endif %} +## Editing advisories in the {% data variables.product.prodname_advisory_database %} -{% ifversion fpt or ghec %} -Only repository owners and administrators can edit repository-level security advisories. For more information, see [AUTOTITLE](/code-security/security-advisories/working-with-repository-security-advisories/editing-a-repository-security-advisory).{% endif %} +{% endif %} 1. Navigate to https://github.com/advisories. 1. Select the security advisory you would like to contribute to.