From 6934bf72d551bb57d940491eef9ab0c0c5900a2e Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 03:31:00 +0530 Subject: [PATCH 01/12] app accessibility changes --- nightwatch/globals.js | 13 +++- src/accessibilityAutomation.js | 135 ++++++++++++++++++++++++++++++++- src/testObservability.js | 1 + src/utils/constants.js | 4 +- src/utils/helper.js | 112 ++++++++++++++++++++++++++- 5 files changed, 256 insertions(+), 9 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 067a724..bc62de7 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -300,7 +300,7 @@ module.exports = { if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } - + process.env.BROWSERSTACK_APP_AUTOMATE = helper.checkTestEnvironmentForAppAutomate(testEnvSettings); // Plugin identifier settings.desiredCapabilities['bstack:options']['browserstackSDK'] = `nightwatch-plugin/${helper.getAgentVersion()}`; @@ -484,8 +484,15 @@ module.exports = { }, async beforeEach(settings) { - browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; - browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; + if (helper.isAppAccessibilitySession()){ + browser.getAccessibilityResults = () => { return accessibilityAutomation.getAppAccessibilityResults(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js + browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAppAccessibilityResultsSummary(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js + } + else{ + browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; + browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; + } + // await accessibilityAutomation.beforeEachExecution(browser); }, // This will be run after each test suite is finished diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 254125c..801bbde 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,6 +1,7 @@ const path = require('path'); const helper = require('./utils/helper'); const Logger = require('./utils/logger'); +const {APP_ALLY_ENDPOINT,APP_ALLY_ISSUES_SUMMARY_ENDPOINT,APP_ALLY_ISSUES_ENDPOINT} = require('./utils/constants'); const util = require('util'); const AccessibilityScripts = require('./scripts/accessibilityScripts'); @@ -161,6 +162,23 @@ class AccessibilityAutomation { return false; } + validateAppA11yCaps(capabilities = {}) { + /* Check if the current driver platform is eligible for AppAccessibility scan */ + Logger.debug(`capabilities ${JSON.stringify(capabilities)}`); + if ( + capabilities?.platformName && + String(capabilities?.platformName).toLowerCase() === 'android' && + capabilities?.platformVersion && + parseInt(capabilities?.platformVersion?.toString()) < 11 + ) { + Logger.warn( + 'App Accessibility Automation tests are supported on OS version 11 and above for Android devices.' + ); + return false; + } + return true; + } + async beforeEachExecution(testMetaData) { try { this.currentTest = browser.currentTest; @@ -168,7 +186,13 @@ class AccessibilityAutomation { testMetaData ); this.currentTest.accessibilityScanStarted = true; - this._isAccessibilitySession = this.validateA11yCaps(browser); + + this._isAppAccessibility = helper.isAppAccessibilitySession(); + if (this._isAppAccessibility) { + this._isAccessibilitySession = this.validateAppA11yCaps(testMetaData.metadata.sessionCapabilities); + } else { + this._isAccessibilitySession = this.validateA11yCaps(browser); + } if (this.isAccessibilityAutomationSession() && browser && this._isAccessibilitySession) { try { @@ -264,10 +288,14 @@ class AccessibilityAutomation { } if (this.currentTest.shouldScanTestForAccessibility === false) { - Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); - + if(commandName && commandName !== '') { + Logger.debug(`Skipping Accessibility scan for command ${commandName} as the test is excluded from accessibility scanning.`); + } else { + Logger.debug('Skipping Accessibility scan as the test is excluded from accessibility scanning.'); + } return; } + try { const browser = browserInstance; @@ -276,6 +304,16 @@ class AccessibilityAutomation { return; } + + if (helper.isAppAccessibilitySession()){ + const results = await browser.executeScript( + helper.formatString(AccessibilityScripts.performScan, JSON.stringify(this.getParamsForAppAccessibility(commandName))), + {} + ); + Logger.debug(util.inspect(results)); + return results; + } + const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { method: commandName || '' }); @@ -291,9 +329,71 @@ class AccessibilityAutomation { } } + async getAppAccessibilityResults(browser) { + if (!helper.isBrowserstackInfra()) { + return []; + } + + if (!helper.isAppAccessibilitySession()) { + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') + return []; + } + + try { + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.issues; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + return result; + } catch (error) { + Logger.error('No accessibility summary was found.'); + Logger.debug(`getAppA11yResults Failed. Error: ${error}`); + return []; + } + + } + + async getAppAccessibilityResultsSummary(browser) { + if (!helper.isBrowserstackInfra()) { + return {}; + } + + if (!helper.isAppAccessibilitySession()) { + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') + return {} + } + try { + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.summary; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + return result; + } catch { + Logger.error('No accessibility summary was found.'); + return {}; + } + } + + async getAppA11yResultResponse(apiUrl, browser, sessionId){ + Logger.debug('Performing scan before getting results/results summary'); + await this.performScan(browser); + + const upperTimeLimit = process.env.BSTACK_A11Y_POLLING_TIMEOUT ? Date.now() + parseInt(process.env.BSTACK_A11Y_POLLING_TIMEOUT) * 1000 : Date.now() + 30000; + const params = { test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now() }; // Query params to pass + const header = { Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}` }; + const apiRespone = await helper.pollApi(apiUrl, params, header, upperTimeLimit); + Logger.debug(`Polling Result: ${JSON.stringify(apiRespone)}`); + return apiRespone; + + } + + async saveAccessibilityResults(browser, dataForExtension = {}) { Logger.debug('Performing scan before saving results'); await this.performScan(browser); + if (helper.isAppAccessibilitySession()){ + return; + } const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); Logger.debug(util.inspect(results)); } @@ -329,7 +429,12 @@ class AccessibilityAutomation { const originalCommandFn = originalCommand.command; originalCommand.command = async function(...args) { - await accessibilityInstance.performScan(browser, commandName); + if ( + !commandName.includes('execute') || + !accessibilityInstance.shouldPatchExecuteScript(args.length ? args[0] : null) + ) { + await accessibilityInstance.performScan(browser, commandName); + } return originalCommandFn.apply(this, args); }; @@ -340,6 +445,28 @@ class AccessibilityAutomation { } } } + + shouldPatchExecuteScript(script) { + if (!script || typeof script !== 'string') { + return true; + } + + return ( + script.toLowerCase().indexOf('browserstack_executor') !== -1 || + script.toLowerCase().indexOf('browserstack_accessibility_automation_script') !== -1 + ); + } + + getParamsForAppAccessibility(commandName) { + return { + 'thTestRunUuid': process.env.TEST_RUN_UUID, + 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, + 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT, + 'authHeader': process.env.BSTACK_A11Y_JWT, + 'scanTimestamp': Date.now(), + 'method': commandName + }; + } } module.exports = AccessibilityAutomation; diff --git a/src/testObservability.js b/src/testObservability.js index 24b8317..964117a 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -234,6 +234,7 @@ class TestObservability { accessibilityScripts.store(); } } + process.env.IS_APP_ACCESSIBILITY = accessibilityAutomation.isAccessibilityAutomationSession() && helper.isAppAutomate(); } diff --git a/src/utils/constants.js b/src/utils/constants.js index 4460c4e..953e360 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -18,7 +18,9 @@ exports.EVENTS = { SCREENSHOT: 'testObservability:screenshot' }; exports.ACCESSIBILITY_URL= 'https://accessibility.browserstack.com/api'; - +exports.APP_ALLY_ENDPOINT = 'https://app-accessibility.browserstack.com/automate'; +exports.APP_ALLY_ISSUES_SUMMARY_ENDPOINT ='api/v1/issues-summary'; +exports.APP_ALLY_ISSUES_ENDPOINT = 'api/v1/issues'; // Maximum size of VCS info which is allowed exports.MAX_GIT_META_DATA_SIZE_IN_BYTES = 64 * 1024; diff --git a/src/utils/helper.js b/src/utils/helper.js index 4ed4667..94a585b 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -16,6 +16,7 @@ const LogPatcher = require('./logPatcher'); const BSTestOpsPatcher = new LogPatcher({}); const sessions = {}; const {execSync} = require('child_process'); +const request = require('@cypress/request'); console = {}; Object.keys(consoleHolder).forEach(method => { @@ -72,6 +73,21 @@ exports.getObservabilityKey = (config, bstackOptions={}) => { return process.env.BROWSERSTACK_ACCESS_KEY || config?.key || bstackOptions?.accessKey; }; +exports.isAppAutomate = () => { + return process.env.BROWSERSTACK_APP_AUTOMATE === 'true'; +}; + +exports.checkTestEnvironmentForAppAutomate = (testEnvSettings) => { + + const firstEnvKey = Object.keys(testEnvSettings)[0]; + const firstEnv = testEnvSettings[firstEnvKey]; + if (firstEnv?.desiredCapabilities?.['appium:options']?.app) { + return true; + } + + return false; +}; + exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; @@ -85,6 +101,10 @@ exports.isTestHubBuild = (pluginSettings = {}, isBuildStart = false) => { } }; +exports.isAppAccessibilitySession = () => { + return process.env.IS_APP_ACCESSIBILITY === 'true'; +}; + exports.isAccessibilityEnabled = (settings) => { if (process.argv.includes('--disable-accessibility')) {return false} @@ -637,7 +657,7 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { if (hostname) { if (hostname.includes('browserstack.com') && !hostname.includes('hub-ft')) { - if (this.isUndefined(caps.browserName)) { + if (this.isAppAutomate()) { product = 'app-automate'; } else { product = 'automate'; @@ -1360,3 +1380,93 @@ exports.logBuildError = (error, product = '') => { } }; +exports.formatString = (template, ...values) => { + let i = 0; + if (template === null) { + return ''; + } + return template.replace(/%s/g, () => { + const value = values[i++]; + return value !== null && value !== undefined ? value : ''; + }); +}; + +exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now()) => { + params.timestamp = Math.round(Date.now() / 1000); + Logger.debug(`current timestamp ${params.timestamp}`); + + try { + const queryString = new URLSearchParams(params).toString(); + const fullUrl = `${url}?${queryString}`; + + const response = await new Promise((resolve, reject) => { + request({ + method: 'GET', + url: fullUrl, + headers: headers, + json: false + }, (error, response, body) => { + if (error) { + reject(error); + } else { + resolve(response); + } + }); + }); + + const responseData = JSON.parse(response.body); + return { + data: responseData, + headers: response.headers, + message: 'Polling succeeded.' + }; + } catch (error) { + if (error.response && error.response.statusCode === 404) { + const nextPollTime = parseInt(error.response.headers.next_poll_time, 10) * 1000; + Logger.debug(`timeInMillis ${nextPollTime}`); + + if (isNaN(nextPollTime)) { + Logger.warn('Invalid or missing `nextPollTime` header. Stopping polling.'); + return { + data: {}, + headers: error.response.headers, + message: 'Invalid nextPollTime header value. Polling stopped.' + }; + } + + const elapsedTime = nextPollTime - Date.now(); + Logger.debug( + `elapsedTime ${elapsedTime} timeInMillis ${nextPollTime} upperLimit ${upperLimit}` + ); + + // Stop polling if the upper time limit is reached + if (nextPollTime > upperLimit) { + Logger.warn('Polling stopped due to upper time limit.'); + return { + data: {}, + headers: error.response.headers, + message: 'Polling stopped due to upper time limit.' + }; + } + + Logger.debug(`Polling again in ${elapsedTime}ms with params:`, params); + + // Wait for the specified time and poll again + await new Promise((resolve) => setTimeout(resolve, elapsedTime)); + return exports.pollApi(url, params, headers, upperLimit, startTime); + } else if (error.response) { + throw { + data: {}, + headers: {}, + message: error.response.body ? JSON.parse(error.response.body).message : 'Unknown error' + }; + } else { + Logger.error(`Unexpected error occurred: ${error}`); + return { data: {}, headers: {}, message: 'Unexpected error occurred.' }; + } + } +}; + + + + From 18258039448208e96135f9b54da9d77a880dacbe Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 1 Dec 2025 16:36:31 +0530 Subject: [PATCH 02/12] minor changes --- nightwatch/globals.js | 11 +++++++---- src/accessibilityAutomation.js | 7 ------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 9f57d1d..46ada9e 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -259,7 +259,7 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser); + process.env.VALID_ALLY_PLATFORM = process.env.BROWSERSTACK_APP_AUTOMATE? accessibilityAutomation.validateAppA11yCaps(test.metadata.sessionCapabilities) : accessibilityAutomation.validateA11yCaps(browser); await accessibilityAutomation.beforeEachExecution(test); if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); @@ -272,7 +272,6 @@ module.exports = { const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); if (TestMap.hasTestFinished(uuid)) { Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); - return; } try { @@ -356,7 +355,9 @@ module.exports = { if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - helper.patchBrowserTerminateCommand(); + if(!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand() + }; } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); @@ -533,7 +534,9 @@ module.exports = { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - helper.patchBrowserTerminateCommand(); + if(!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand() + }; } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 4458854..6431d65 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -165,7 +165,6 @@ class AccessibilityAutomation { validateAppA11yCaps(capabilities = {}) { /* Check if the current driver platform is eligible for AppAccessibility scan */ - Logger.debug(`capabilities ${JSON.stringify(capabilities)}`); if ( capabilities?.platformName && String(capabilities?.platformName).toLowerCase() === 'android' && @@ -286,16 +285,10 @@ class AccessibilityAutomation { if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); - return; } if (this.currentTest.shouldScanTestForAccessibility === false) { - if(commandName && commandName !== '') { - Logger.debug(`Skipping Accessibility scan for command ${commandName} as the test is excluded from accessibility scanning.`); - } else { - Logger.debug('Skipping Accessibility scan as the test is excluded from accessibility scanning.'); - } return; } From c52b4b8dd68a084f98216adf1ed55cedd2d3da47 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 1 Dec 2025 22:50:38 +0530 Subject: [PATCH 03/12] review changes pt.1 & lint fixes --- nightwatch/globals.js | 16 ++++---- src/accessibilityAutomation.js | 70 ++++++++++++++++++++-------------- src/utils/helper.js | 9 ++++- 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 46ada9e..a62adb2 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -272,6 +272,7 @@ module.exports = { const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); if (TestMap.hasTestFinished(uuid)) { Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); + return; } try { @@ -355,8 +356,8 @@ module.exports = { if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - if(!process.env.BROWSERSTACK_APP_AUTOMATE){ - helper.patchBrowserTerminateCommand() + if (!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand(); }; } } catch (err){ @@ -486,10 +487,9 @@ module.exports = { async beforeEach(settings) { if (helper.isAppAccessibilitySession()){ - browser.getAccessibilityResults = () => { return accessibilityAutomation.getAppAccessibilityResults(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js - browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAppAccessibilityResultsSummary(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js - } - else{ + browser.getAccessibilityResults = () => { return accessibilityAutomation.getAppAccessibilityResults(browser) }; + browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAppAccessibilityResultsSummary(browser) }; + } else { browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; } @@ -534,8 +534,8 @@ module.exports = { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - if(!process.env.BROWSERSTACK_APP_AUTOMATE){ - helper.patchBrowserTerminateCommand() + if (!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand(); }; } } catch (err){ diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index a1b8e5f..e2feb3f 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,7 +1,7 @@ const path = require('path'); const helper = require('./utils/helper'); const Logger = require('./utils/logger'); -const {APP_ALLY_ENDPOINT,APP_ALLY_ISSUES_SUMMARY_ENDPOINT,APP_ALLY_ISSUES_ENDPOINT} = require('./utils/constants'); +const {APP_ALLY_ENDPOINT, APP_ALLY_ISSUES_SUMMARY_ENDPOINT, APP_ALLY_ISSUES_ENDPOINT} = require('./utils/constants'); const util = require('util'); const AccessibilityScripts = require('./scripts/accessibilityScripts'); @@ -166,16 +166,18 @@ class AccessibilityAutomation { validateAppA11yCaps(capabilities = {}) { /* Check if the current driver platform is eligible for AppAccessibility scan */ if ( - capabilities?.platformName && + capabilities?.platformName && String(capabilities?.platformName).toLowerCase() === 'android' && capabilities?.platformVersion && parseInt(capabilities?.platformVersion?.toString()) < 11 ) { - Logger.warn( - 'App Accessibility Automation tests are supported on OS version 11 and above for Android devices.' - ); - return false; + Logger.warn( + 'App Accessibility Automation tests are supported on OS version 11 and above for Android devices.' + ); + + return false; } + return true; } @@ -285,6 +287,7 @@ class AccessibilityAutomation { if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); + return; } @@ -307,6 +310,7 @@ class AccessibilityAutomation { {} ); Logger.debug(util.inspect(results)); + return results; } AccessibilityAutomation.pendingAllyReq++; @@ -333,20 +337,23 @@ class AccessibilityAutomation { } if (!helper.isAppAccessibilitySession()) { - Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') - return []; + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results.'); + + return []; } try { - const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; - const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); - const result = apiRespone?.data?.data?.issues; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); - return result; + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.issues; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + + return result; } catch (error) { - Logger.error('No accessibility summary was found.'); - Logger.debug(`getAppA11yResults Failed. Error: ${error}`); - return []; + Logger.error('No accessibility results were found.'); + Logger.debug(`getAppAccessibilityResults Failed. Error: ${error}`); + + return []; } } @@ -357,18 +364,22 @@ class AccessibilityAutomation { } if (!helper.isAppAccessibilitySession()) { - Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') - return {} + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.'); + + return {}; } try { - const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; - const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); - const result = apiRespone?.data?.data?.summary; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); - return result; - } catch { - Logger.error('No accessibility summary was found.'); - return {}; + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.summary; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + + return result; + } catch (error) { + Logger.error('No accessibility result summary were found.'); + Logger.debug(`getAppAccessibilityResultsSummary Failed. Error: ${error}`); + + return {}; } } @@ -377,10 +388,11 @@ class AccessibilityAutomation { await this.performScan(browser); const upperTimeLimit = process.env.BSTACK_A11Y_POLLING_TIMEOUT ? Date.now() + parseInt(process.env.BSTACK_A11Y_POLLING_TIMEOUT) * 1000 : Date.now() + 30000; - const params = { test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now() }; // Query params to pass - const header = { Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}` }; + const params = {test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now()}; // Query params to pass + const header = {Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}`}; const apiRespone = await helper.pollApi(apiUrl, params, header, upperTimeLimit); Logger.debug(`Polling Result: ${JSON.stringify(apiRespone)}`); + return apiRespone; } @@ -390,7 +402,7 @@ class AccessibilityAutomation { Logger.debug('Performing scan before saving results'); await this.performScan(browser); if (helper.isAppAccessibilitySession()){ - return; + return; } const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); diff --git a/src/utils/helper.js b/src/utils/helper.js index 23d5b8a..8f5b519 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1410,8 +1410,10 @@ exports.formatString = (template, ...values) => { if (template === null) { return ''; } + return template.replace(/%s/g, () => { const value = values[i++]; + return value !== null && value !== undefined ? value : ''; }); }; @@ -1440,6 +1442,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( }); const responseData = JSON.parse(response.body); + return { data: responseData, headers: response.headers, @@ -1452,6 +1455,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( if (isNaN(nextPollTime)) { Logger.warn('Invalid or missing `nextPollTime` header. Stopping polling.'); + return { data: {}, headers: error.response.headers, @@ -1467,6 +1471,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( // Stop polling if the upper time limit is reached if (nextPollTime > upperLimit) { Logger.warn('Polling stopped due to upper time limit.'); + return { data: {}, headers: error.response.headers, @@ -1478,6 +1483,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( // Wait for the specified time and poll again await new Promise((resolve) => setTimeout(resolve, elapsedTime)); + return exports.pollApi(url, params, headers, upperLimit, startTime); } else if (error.response) { throw { @@ -1487,7 +1493,8 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( }; } else { Logger.error(`Unexpected error occurred: ${error}`); - return { data: {}, headers: {}, message: 'Unexpected error occurred.' }; + + return {data: {}, headers: {}, message: 'Unexpected error occurred.'}; } } }; From 3b795f6af380203805146a05178241fb182f2295 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 2 Dec 2025 14:20:25 +0530 Subject: [PATCH 04/12] minor change --- .eslintrc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index 09780ca..9ee7cbc 100644 --- a/.eslintrc +++ b/.eslintrc @@ -108,6 +108,8 @@ "by": "readonly", "expect": "readonly", "browser": "readonly", - "Key": "readonly" + "Key": "readonly", + "URLSearchParams": "readonly" + } } From 143d0723f7e1eae02b16c09c5147f3eec47a8b87 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 2 Dec 2025 17:47:45 +0530 Subject: [PATCH 05/12] lint fix --- nightwatch/globals.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index a62adb2..19908d4 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -259,7 +259,7 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - process.env.VALID_ALLY_PLATFORM = process.env.BROWSERSTACK_APP_AUTOMATE? accessibilityAutomation.validateAppA11yCaps(test.metadata.sessionCapabilities) : accessibilityAutomation.validateA11yCaps(browser); + process.env.VALID_ALLY_PLATFORM = process.env.BROWSERSTACK_APP_AUTOMATE ? accessibilityAutomation.validateAppA11yCaps(test.metadata.sessionCapabilities) : accessibilityAutomation.validateA11yCaps(browser); await accessibilityAutomation.beforeEachExecution(test); if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); From 6956b301c809445cda87801d238caf2a45f92355 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 3 Dec 2025 19:28:30 +0530 Subject: [PATCH 06/12] added handling of a edge case --- nightwatch/globals.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 19908d4..ab03384 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -21,6 +21,7 @@ const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; let testRunner = ''; +let testEventPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -264,7 +265,7 @@ module.exports = { if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; - await testObservability.sendTestRunEvent('TestRunStarted', test, uuid); + testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); } }); @@ -278,7 +279,7 @@ module.exports = { try { await accessibilityAutomation.afterEachExecution(test, uuid); if (testRunner !== 'cucumber'){ - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid); + testEventPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); TestMap.markTestFinished(uuid); } @@ -474,6 +475,10 @@ module.exports = { await helper.deleteRerunFile(); } try { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } await testObservability.stopBuildUpstream(); if (process.env.BROWSERSTACK_TESTHUB_UUID) { Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BROWSERSTACK_TESTHUB_UUID} to view build report, insights, and many more debugging information all at one place!\n`); @@ -543,6 +548,13 @@ module.exports = { } addProductMapAndbuildUuidCapability(settings); + }, + + async afterChildProcess() { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } } }; From d36ada5d9cf4d87906714ec15168860070972e1d Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 4 Dec 2025 15:39:20 +0530 Subject: [PATCH 07/12] fixed the polling logic --- src/accessibilityAutomation.js | 6 ++--- src/utils/helper.js | 40 ++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index e2feb3f..7e914eb 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -346,7 +346,7 @@ class AccessibilityAutomation { const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); const result = apiRespone?.data?.data?.issues; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + Logger.debug(`Results: ${JSON.stringify(result)}`); return result; } catch (error) { @@ -372,7 +372,7 @@ class AccessibilityAutomation { const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); const result = apiRespone?.data?.data?.summary; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + Logger.debug(`Results Summary: ${JSON.stringify(result)}`); return result; } catch (error) { @@ -391,7 +391,7 @@ class AccessibilityAutomation { const params = {test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now()}; // Query params to pass const header = {Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}`}; const apiRespone = await helper.pollApi(apiUrl, params, header, upperTimeLimit); - Logger.debug(`Polling Result: ${JSON.stringify(apiRespone)}`); + Logger.debug(`Polling Result: ${JSON.stringify(apiRespone.message)}`); return apiRespone; diff --git a/src/utils/helper.js b/src/utils/helper.js index 8f5b519..acd643d 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1442,50 +1442,52 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( }); const responseData = JSON.parse(response.body); - - return { - data: responseData, - headers: response.headers, - message: 'Polling succeeded.' - }; - } catch (error) { - if (error.response && error.response.statusCode === 404) { - const nextPollTime = parseInt(error.response.headers.next_poll_time, 10) * 1000; - Logger.debug(`timeInMillis ${nextPollTime}`); + + if (response.statusCode === 404) { + const nextPollTime = parseInt(response.headers?.next_poll_time, 10) * 1000; + Logger.debug(`nextPollTime: ${nextPollTime}`); if (isNaN(nextPollTime)) { Logger.warn('Invalid or missing `nextPollTime` header. Stopping polling.'); return { data: {}, - headers: error.response.headers, + headers: response.headers || {}, message: 'Invalid nextPollTime header value. Polling stopped.' }; } - const elapsedTime = nextPollTime - Date.now(); - Logger.debug( - `elapsedTime ${elapsedTime} timeInMillis ${nextPollTime} upperLimit ${upperLimit}` - ); - // Stop polling if the upper time limit is reached if (nextPollTime > upperLimit) { Logger.warn('Polling stopped due to upper time limit.'); return { data: {}, - headers: error.response.headers, + headers: response.headers || {}, message: 'Polling stopped due to upper time limit.' }; } - Logger.debug(`Polling again in ${elapsedTime}ms with params:`, params); + const elapsedTime = Math.max(0, nextPollTime - Date.now()); + Logger.debug( + `elapsedTime ${elapsedTime} nextPollTimes ${nextPollTime} upperLimit ${upperLimit}` + ); + + Logger.debug(`Polling for results again in ${elapsedTime}ms`); // Wait for the specified time and poll again await new Promise((resolve) => setTimeout(resolve, elapsedTime)); return exports.pollApi(url, params, headers, upperLimit, startTime); - } else if (error.response) { + } + + return { + data: responseData, + headers: response.headers, + message: 'Polling succeeded.' + }; + } catch (error) { + if (error.response) { throw { data: {}, headers: {}, From 906fc8796f997ddb3080bb50cc469365953b5832 Mon Sep 17 00:00:00 2001 From: bhargavi vaidya Date: Tue, 9 Dec 2025 14:01:23 +0000 Subject: [PATCH 08/12] Build Unification changes for TRA and Accessibility (#44) * TRA changes * changes for testMap implementation * added EOF lines * TRA changes * TRA changes pt.3 * fix: lint error * accessibility changes * fix: lint error * minor change * minor change * fix:lint errors * added null checks * review changes pt.1 * minor change * fix: static testMap added * minor change * minor change in try-catch * temp: fallback for old core version * eslintrc change * fix for double test events in cucumber runner * env var name changed * minor changes * removed the fallback and added alternative * review changes pt.2 * Update src/utils/testMap.js Co-authored-by: Amaan Hakim <89768375+amaanbs@users.noreply.github.com> * review changes pt.3 * review changes pt.4 * minor change * fix: lint issues * fixed the UTs * minor change * review changes pt.5 * fixed lint issues * minor log change * minor change * fix for wrong product map * minor change * fix for handling a edge case * fix for the timeout issue --------- Co-authored-by: Amaan Hakim <89768375+amaanbs@users.noreply.github.com> --- .eslintrc | 1 + nightwatch/globals.js | 107 +++-- src/accessibilityAutomation.js | 448 +++++------------- src/scripts/accessibilityScripts.js | 107 +++++ src/testObservability.js | 284 ++++++++--- src/testorchestration/orchestrationUtils.js | 6 +- src/testorchestration/requestUtils.js | 4 +- src/utils/crashReporter.js | 2 +- src/utils/helper.js | 114 ++++- src/utils/logPatcher.js | 2 +- src/utils/requestQueueHandler.js | 4 +- src/utils/testMap.js | 95 ++++ .../test-observability/processTestRunData.js | 21 +- test/src/utils/helper.js | 31 +- 14 files changed, 757 insertions(+), 469 deletions(-) create mode 100644 src/scripts/accessibilityScripts.js create mode 100644 src/utils/testMap.js diff --git a/.eslintrc b/.eslintrc index fc74d48..09780ca 100644 --- a/.eslintrc +++ b/.eslintrc @@ -102,6 +102,7 @@ "Promise": true, "Proxy": true, "Set": true, + "Map": true, "Reflect": true, "element": "readonly", "by": "readonly", diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 38ade3e..ecc9e65 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -9,6 +9,7 @@ const path = require('path'); const AccessibilityAutomation = require('../src/accessibilityAutomation'); const eventHelper = require('../src/utils/eventHelper'); const OrchestrationUtils = require('../src/testorchestration/orchestrationUtils'); +const TestMap = require('../src/utils/testMap'); const localTunnel = new LocalTunnel(); const testObservability = new TestObservability(); const accessibilityAutomation = new AccessibilityAutomation(); @@ -19,6 +20,8 @@ const _tests = {}; const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; +let testRunner = ''; +let testEventPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -65,8 +68,8 @@ module.exports = { promises.push(testObservability.processTestReportFile(JSON.parse(JSON.stringify(modulesWithEnv[testSetting][testFile])))); } } - await Promise.all(promises); + done(); } catch (error) { CrashReporter.uploadCrashReport(error.message, error.stack); @@ -92,7 +95,7 @@ module.exports = { const gherkinDocument = reportData?.gherkinDocument.find((document) => document.uri === pickleData.uri); const featureData = gherkinDocument.feature; const uniqueId = uuidv4(); - process.env.TEST_OPS_TEST_UUID = uniqueId; + process.env.TEST_RUN_UUID = uniqueId; Object.values(workerList).forEach((worker) => { worker.process.on('message', async (data) => { @@ -253,15 +256,37 @@ module.exports = { eventBroadcaster.on('ScreenshotCreated', async (args) => { if (!helper.isTestObservabilitySession()) {return} - handleScreenshotUpload({args: args, uuid: process.env.TEST_OPS_TEST_UUID}); + handleScreenshotUpload({args: args, uuid: process.env.TEST_RUN_UUID}); }); eventBroadcaster.on('TestRunStarted', async (test) => { + process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser); await accessibilityAutomation.beforeEachExecution(test); + if (testRunner !== 'cucumber'){ + const uuid = TestMap.storeTestDetails(test); + process.env.TEST_RUN_UUID = uuid; + testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); + } }); eventBroadcaster.on('TestRunFinished', async (test) => { - await accessibilityAutomation.afterEachExecution(test); + const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); + if (TestMap.hasTestFinished(uuid)) { + Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); + + return; + } + try { + await accessibilityAutomation.afterEachExecution(test, uuid); + if (testRunner !== 'cucumber'){ + testEventPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); + TestMap.markTestFinished(uuid); + } + + } catch (error) { + Logger.error(`Error in TestRunFinished event: ${error.message}`); + TestMap.markTestFinished(uuid); + } }); }, @@ -272,10 +297,12 @@ module.exports = { }, async before(settings, testEnvSettings) { + testRunner = settings.test_runner.type; + const pluginSettings = settings['@nightwatch/browserstack']; if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } - + process.env.BROWSERSTACK_APP_AUTOMATE = helper.checkTestEnvironmentForAppAutomate(testEnvSettings); // Plugin identifier settings.desiredCapabilities['bstack:options']['browserstackSDK'] = `nightwatch-plugin/${helper.getAgentVersion()}`; @@ -295,6 +322,7 @@ module.exports = { try { testObservability.configure(settings); + accessibilityAutomation.configure(settings); if (helper.isTestObservabilitySession()) { if (settings.reporter_options) { if (settings.reporter_options['save_command_result_value'] !== true){ @@ -312,7 +340,7 @@ module.exports = { settings.test_runner.options['require'] = path.resolve(__dirname, 'observabilityLogPatcherHook.js'); } settings.globals['customReporterCallbackTimeout'] = CUSTOM_REPORTER_CALLBACK_TIMEOUT; - if (testObservability._user && testObservability._key) { + if (helper.isTestHubBuild(pluginSettings, true)) { await testObservability.launchTestSession(); } if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS && process.env.BROWSERSTACK_RERUN_TESTS!=='null') { @@ -323,7 +351,17 @@ module.exports = { } catch (error) { Logger.error(`Could not configure or launch test reporting and analytics - ${error}`); } - + + try { + // In parallel mode, env-specific settings are passed to beforeChildProcess hook instead of before hook, + if (helper.isAccessibilitySession() && !settings.parallel_mode) { + accessibilityAutomation.setAccessibilityCapabilities(settings); + accessibilityAutomation.commandWrapper(); + helper.patchBrowserTerminateCommand(); + } + } catch (err){ + Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); + } // Initialize and configure test orchestration try { if (helper.isTestObservabilitySession()) { @@ -408,22 +446,6 @@ module.exports = { Logger.error(`Could not configure test orchestration - ${error}`); } - try { - accessibilityAutomation.configure(settings); - if (helper.isAccessibilitySession()) { - if (accessibilityAutomation._user && accessibilityAutomation._key) { - const [jwtToken, testRunId] = await accessibilityAutomation.createAccessibilityTestRun(); - process.env.BS_A11Y_JWT = jwtToken; - process.env.BS_A11Y_TEST_RUN_ID = testRunId; - if (helper.isAccessibilitySession()) { - accessibilityAutomation.setAccessibilityCapabilities(settings); - } - } - } - } catch (error) { - Logger.error(`Could not configure or launch accessibility automation - ${error}`); - } - addProductMapAndbuildUuidCapability(settings); }, @@ -443,46 +465,40 @@ module.exports = { } catch (error) { Logger.error(`Error collecting build data for test orchestration: ${error}`); } - - if (helper.isTestObservabilitySession()) { + + if (helper.isTestHubBuild()) { process.env.NIGHTWATCH_RERUN_FAILED = nightwatchRerun; process.env.NIGHTWATCH_RERUN_REPORT_FILE = nightwatchRerunFile; if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS) { await helper.deleteRerunFile(); } try { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } await testObservability.stopBuildUpstream(); - if (process.env.BS_TESTOPS_BUILD_HASHED_ID) { - Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID} to view build report, insights, and many more debugging information all at one place!\n`); + if (process.env.BROWSERSTACK_TESTHUB_UUID) { + Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BROWSERSTACK_TESTHUB_UUID} to view build report, insights, and many more debugging information all at one place!\n`); } } catch (error) { Logger.error(`Something went wrong in stopping build session for test reporting and analytics - ${error}`); } process.exit(); } - if (helper.isAccessibilitySession()){ - try { - await accessibilityAutomation.stopAccessibilityTestRun(); - } catch (error) { - Logger.error(`Exception in stop accessibility test run: ${error}`); - } - - } }, async beforeEach(settings) { browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; - // await accessibilityAutomation.beforeEachExecution(browser); }, // This will be run after each test suite is finished async afterEach(settings) { - // await accessibilityAutomation.afterEachExecution(browser); }, beforeChildProcess(settings) { - + if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } @@ -514,12 +530,23 @@ module.exports = { try { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); + accessibilityAutomation.commandWrapper(); + helper.patchBrowserTerminateCommand(); } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); } addProductMapAndbuildUuidCapability(settings); + }, + + async afterChildProcess() { + + await helper.shutDownRequestHandler(); + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } } }; @@ -569,10 +596,10 @@ const addProductMapAndbuildUuidCapability = (settings) => { if (settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options']['buildProductMap'] = buildProductMap; - settings.desiredCapabilities['bstack:options']['testhubBuildUuid'] = process.env.BS_TESTOPS_BUILD_HASHED_ID ? process.env.BS_TESTOPS_BUILD_HASHED_ID : '' ; + settings.desiredCapabilities['bstack:options']['testhubBuildUuid'] = process.env.BROWSERSTACK_TESTHUB_UUID ? process.env.BROWSERSTACK_TESTHUB_UUID : '' ; } else { settings.desiredCapabilities['browserstack.buildProductMap'] = buildProductMap; - settings.desiredCapabilities['browserstack.testhubBuildUuid'] = process.env.BS_TESTOPS_BUILD_HASHED_ID ? process.env.BS_TESTOPS_BUILD_HASHED_ID : '' ; + settings.desiredCapabilities['browserstack.testhubBuildUuid'] = process.env.BROWSERSTACK_TESTHUB_UUID ? process.env.BROWSERSTACK_TESTHUB_UUID : '' ; } } catch (error) { Logger.debug(`Error while sending productmap and build capabilities ${error}`); diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index af97fa0..d8f75c1 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,28 +1,13 @@ const path = require('path'); const helper = require('./utils/helper'); -const {makeRequest} = require('./utils/requestHelper'); const Logger = require('./utils/logger'); -const {ACCESSIBILITY_URL} = require('./utils/constants'); const util = require('util'); +const AccessibilityScripts = require('./scripts/accessibilityScripts'); class AccessibilityAutomation { + static pendingAllyReq = 0; configure(settings = {}) { this._settings = settings['@nightwatch/browserstack'] || {}; - - if (this._settings.accessibility) { - process.env.BROWSERSTACK_ACCESSIBILITY = - String(this._settings.accessibility).toLowerCase() === 'true'; - } - if (process.argv.includes('--disable-accessibility')) { - process.env.BROWSERSTACK_ACCESSIBILITY = false; - - return; - } - process.env.BROWSERSTACK_INFRA = true; - if (settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1){ - process.env.BROWSERSTACK_INFRA = false; - } - this._testRunner = settings.test_runner; this._bstackOptions = {}; if ( @@ -37,165 +22,11 @@ class AccessibilityAutomation { this._user = helper.getUserName(settings, this._settings); this._key = helper.getAccessKey(settings, this._settings); } - } - - async createAccessibilityTestRun() { - const userName = this._user; - const accessKey = this._key; - - if (helper.isUndefined(userName) || helper.isUndefined(accessKey)) { - Logger.error( - 'Exception while creating test run for BrowserStack Accessibility Automation: Missing authentication token' - ); - - return [null, null]; - } - - try { - let accessibilityOptions; - if (helper.isUndefined(this._settings.accessibilityOptions)) { - accessibilityOptions = {}; - } else { - accessibilityOptions = this._settings.accessibilityOptions; - } - - const fromProduct = { - accessibility: true - }; - - const data = { - projectName: helper.getProjectName(this._settings, this._bstackOptions, fromProduct), - buildName: - helper.getBuildName(this._settings, this._bstackOptions, fromProduct) || - path.basename(path.resolve(process.cwd())), - startTime: new Date().toISOString(), - description: accessibilityOptions.buildDescription || '', - source: { - frameworkName: helper.getFrameworkName(this._testRunner), - frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion() - }, - settings: accessibilityOptions, - versionControl: await helper.getGitMetaData(), - ciInfo: helper.getCiInfo(), - hostInfo: helper.getHostInfo(), - browserstackAutomation: helper.isBrowserstackInfra() - }; - const config = { - auth: { - user: userName, - pass: accessKey - }, - headers: { - 'Content-Type': 'application/json' - } - }; - - const response = await makeRequest('POST', 'test_runs', data, config, ACCESSIBILITY_URL); - const responseData = response.data.data || {}; - - accessibilityOptions.scannerVersion = responseData.scannerVersion; - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); - - return [responseData.accessibilityToken, responseData.id]; - } catch (error) { - process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - if (error.response) { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.response.status - } ${error.response.statusText} ${JSON.stringify(error.response.data)}` - ); - } else { - if (error.message === 'Invalid configuration passed.') { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.message || error.stack - }` - ); - for (const errorkey of error.errors) { - Logger.error(errorkey.message); - } - process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - } else { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.message || error.stack - }` - ); - } - } - - return [null, null]; - } - } - - async stopAccessibilityTestRun() { - if ( - helper.isUndefined(process.env.BS_A11Y_JWT) || - typeof process.env.BS_A11Y_JWT !== 'string' - ) { - return { - status: 'error', - message: 'Build creation had failed.' - }; - } - - const data = {endTime: new Date().toISOString()}; - const config = { - headers: { - Authorization: `Bearer ${process.env.BS_A11Y_JWT}`, - 'Content-Type': 'application/json' - } - }; - const options = { - ...config, - ...{ - body: data, - auth: null, - json: true - } - }; - try { - const response = await makeRequest( - 'PUT', - 'test_runs/stop', - options, - config, - ACCESSIBILITY_URL - ); - if (response.data.error) { - throw new Error('Invalid request: ' + response.data.error); - } else { - Logger.info( - `BrowserStack Accessibility Automation Test Run marked as completed at ${new Date().toISOString()}` - ); - - return {status: 'success', message: ''}; - } - } catch (error) { - if (error.response) { - Logger.error( - `Exception while marking completion of BrowserStack Accessibility Automation Test Run: ${ - error.response.status - } ${error.response.statusText} ${JSON.stringify(error.response.data)}` - ); - } else { - Logger.error( - `Exception while marking completion of BrowserStack Accessibility Automation Test Run: ${ - error.message || util.format(error) - }` - ); - } - - return { - status: 'error', - message: - error.message || - (error.response ? `${error.response.status}:${error.response.statusText}` : error) - }; - } + const accessibilityOptions = helper.isUndefined(this._settings.accessibilityOptions) + ? {} + : this._settings.accessibilityOptions; + process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); } setAccessibilityCapabilities(settings) { @@ -223,25 +54,21 @@ class AccessibilityAutomation { if (this._bstackOptions) { this._bstackOptions.accessibility = this._settings.accessibility; if (this._bstackOptions.accessibilityOptions) { - this._bstackOptions.accessibilityOptions.authToken = process.env.BS_A11Y_JWT; + this._bstackOptions.accessibilityOptions.authToken = process.env.BSTACK_A11Y_JWT; } else { - this._bstackOptions.accessibilityOptions = {authToken: process.env.BS_A11Y_JWT}; + this._bstackOptions.accessibilityOptions = {authToken: process.env.BSTACK_A11Y_JWT}; } - this._bstackOptions.accessibilityOptions.scannerVersion = JSON.parse( - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS - ).scannerVersion; + this._bstackOptions.accessibilityOptions.scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } else if (settings.desiredCapabilities['browserstack.accessibility']) { if (settings.desiredCapabilities['browserstack.accessibilityOptions']) { settings.desiredCapabilities['browserstack.accessibilityOptions'].authToken = - process.env.BS_A11Y_JWT; + process.env.BSTACK_A11Y_JWT; } else { settings.desiredCapabilities['browserstack.accessibilityOptions'] = { - authToken: process.env.BS_A11Y_JWT + authToken: process.env.BSTACK_A11Y_JWT }; } - settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = JSON.parse( - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS - ).scannerVersion; + settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } } } catch (e) { @@ -256,9 +83,9 @@ class AccessibilityAutomation { } const isBrowserstackAccessibilityEnabled = process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; const hasA11yJwtToken = - typeof process.env.BS_A11Y_JWT === 'string' && - process.env.BS_A11Y_JWT.length > 0 && - process.env.BS_A11Y_JWT !== 'null'; + typeof process.env.BSTACK_A11Y_JWT === 'string' && + process.env.BSTACK_A11Y_JWT.length > 0 && + process.env.BSTACK_A11Y_JWT !== 'null'; return isBrowserstackAccessibilityEnabled && hasA11yJwtToken; } catch (error) { @@ -297,23 +124,7 @@ class AccessibilityAutomation { return false; } - fetchPlatformDetails(driver) { - let response = {}; - try { - response = { - os_name: driver.capabilities.platformName, - os_version: helper.getPlatformVersion(driver), - browser_name: driver.capabilities.browserName, - browser_version: driver.capabilities.browserVersion - }; - } catch (error) { - Logger.debug(`Exception in fetching platform details with error : ${error}`); - } - - return response; - } - - setExtension(driver) { + validateA11yCaps(driver) { try { const capabilityConfig = driver.desiredCapabilities || {}; const deviceName = driver.capabilities.deviceName || (capabilityConfig['bstack:options'] ? capabilityConfig['bstack:options'].deviceName : capabilityConfig.device) || ''; @@ -345,7 +156,7 @@ class AccessibilityAutomation { return true; } catch (error) { - Logger.debug(`Exception in setExtension Error: ${error}`); + Logger.debug(`Exception in validateA11yCaps Error: ${error}`); } return false; @@ -358,55 +169,14 @@ class AccessibilityAutomation { testMetaData ); this.currentTest.accessibilityScanStarted = true; - this._isAccessibilitySession = this.setExtension(browser); - - if (this.isAccessibilityAutomationSession() && browser && helper.isAccessibilitySession() && this._isAccessibilitySession) { - try { - const session = await browser.session(); - if (session) { - let pageOpen = true; - const currentURL = await browser.driver.getCurrentUrl(); - - let url = {}; - try { - url = new URL(currentURL); - pageOpen = true; - } catch (e) { - pageOpen = false; - } - pageOpen = url.protocol === 'http:' || url.protocol === 'https:'; - - if (pageOpen) { - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info( - 'Setup for Accessibility testing has started. Automate test case execution will begin momentarily.' - ); - - await browser.executeAsyncScript(` - const callback = arguments[arguments.length - 1]; - const fn = () => { - window.addEventListener('A11Y_TAP_STARTED', fn2); - const e = new CustomEvent('A11Y_FORCE_START'); - window.dispatchEvent(e); - }; - const fn2 = () => { - window.removeEventListener('A11Y_TAP_STARTED', fn); - callback(); - } - fn(); - `); - } else { - await browser.executeAsyncScript(` - const e = new CustomEvent('A11Y_FORCE_STOP'); - window.dispatchEvent(e); - `); - } - } - this.currentTest.accessibilityScanStarted = + this._isAccessibilitySession = this.validateA11yCaps(browser); + + if (this.isAccessibilityAutomationSession() && browser && this._isAccessibilitySession) { + try { + this.currentTest.accessibilityScanStarted = this.currentTest.shouldScanTestForAccessibility; - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info('Automate test case execution has started.'); - } + if (this.currentTest.shouldScanTestForAccessibility) { + Logger.info('Automate test case execution has started.'); } } catch (e) { Logger.error('Exception in starting accessibility automation scan for this test case', e); @@ -417,46 +187,23 @@ class AccessibilityAutomation { } } - async afterEachExecution(testMetaData) { + async afterEachExecution(testMetaData, uuid) { try { if (this.currentTest.accessibilityScanStarted && this.isAccessibilityAutomationSession() && this._isAccessibilitySession) { if (this.currentTest.shouldScanTestForAccessibility) { Logger.info( 'Automate test case execution has ended. Processing for accessibility testing is underway. ' ); - } - const dataForExtension = { - saveResults: this.currentTest.shouldScanTestForAccessibility, - testDetails: { - name: testMetaData.testcase, - testRunId: process.env.BS_A11Y_TEST_RUN_ID, - filePath: testMetaData.metadata.modulePath, - scopeList: [testMetaData.metadata.name, testMetaData.testcase] - }, - platform: await this.fetchPlatformDetails(browser) - }; - const final_res = await browser.executeAsyncScript( - ` - const callback = arguments[arguments.length - 1]; - - this.res = null; - if (arguments[0].saveResults) { - window.addEventListener('A11Y_TAP_TRANSPORTER', (event) => { - window.tapTransporterData = event.detail; - this.res = window.tapTransporterData; - callback(this.res); - }); - } - const e = new CustomEvent('A11Y_TEST_END', {detail: arguments[0]}); - window.dispatchEvent(e); - if (arguments[0].saveResults !== true ) { - callback(); - } - `, - dataForExtension - ); - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info('Accessibility testing for this test case has ended.'); + + const dataForExtension = { + 'thTestRunUuid': uuid, + 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, + 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT + }; + AccessibilityAutomation.pendingAllyReq++; + await this.saveAccessibilityResults(browser, dataForExtension); + AccessibilityAutomation.pendingAllyReq--; + Logger.info('Accessibility testing for this test case has ended.'); } } } catch (er) { @@ -474,21 +221,9 @@ class AccessibilityAutomation { return {}; } try { - const results = await browser.executeScript(` - return new Promise(function (resolve, reject) { - try { - const event = new CustomEvent('A11Y_TAP_GET_RESULTS'); - const fn = function (event) { - window.removeEventListener('A11Y_RESULTS_RESPONSE', fn); - resolve(event.detail.data); - }; - window.addEventListener('A11Y_RESULTS_RESPONSE', fn); - window.dispatchEvent(event); - } catch { - reject(); - } - }); - `); + Logger.debug('Performing scan before getting results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.getResults); return results; } catch { @@ -507,21 +242,9 @@ class AccessibilityAutomation { return {}; } try { - const summaryResults = await browser.executeScript(` - return new Promise(function (resolve, reject) { - try{ - const event = new CustomEvent('A11Y_TAP_GET_RESULTS_SUMMARY'); - const fn = function (event) { - window.removeEventListener('A11Y_RESULTS_SUMMARY_RESPONSE', fn); - resolve(event.detail.summary); - }; - window.addEventListener('A11Y_RESULTS_SUMMARY_RESPONSE', fn); - window.dispatchEvent(event); - } catch { - reject(); - } - }); - `); + Logger.debug('Performing scan before getting results summary'); + await this.performScan(browser); + const summaryResults = await browser.executeAsyncScript(AccessibilityScripts.getResultsSummary); return summaryResults; } catch { @@ -535,6 +258,95 @@ class AccessibilityAutomation { return Object.fromEntries(Object.entries(accessibilityOptions).filter(([k, v]) => !(k.toLowerCase() === 'excludetagsintestingscope' || k.toLowerCase() === 'includetagsintestingscope'))); } + async performScan(browserInstance = null, commandName = '') { + + if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { + Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); + + return; + } + + if (this.currentTest.shouldScanTestForAccessibility === false) { + Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); + + return; + } + try { + const browser = browserInstance; + + if (!browser) { + Logger.error('No browser instance available for accessibility scan'); + + return; + } + AccessibilityAutomation.pendingAllyReq++; + const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { + method: commandName || '' + }); + AccessibilityAutomation.pendingAllyReq--; + Logger.debug(util.inspect(results)); + + return results; + + } catch (err) { + AccessibilityAutomation.pendingAllyReq--; + Logger.error('Accessibility Scan could not be performed: ' + err.message); + Logger.debug('Stack trace:', err.stack); + + return; + } + } + + async saveAccessibilityResults(browser, dataForExtension = {}) { + Logger.debug('Performing scan before saving results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); + + return results; + } + + async commandWrapper() { + const nightwatchMain = require.resolve('nightwatch'); + const nightwatchDir = path.dirname(nightwatchMain); + + const commandJson = AccessibilityScripts.commandsToWrap; + const accessibilityInstance = this; + for (const commandKey in commandJson) { + if (commandJson[commandKey].method === 'protocolAction'){ + commandJson[commandKey].name.forEach(commandName => { + try { + const commandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const OriginalClass = require(commandPath); + const originalProtocolAction = OriginalClass.prototype.protocolAction; + + OriginalClass.prototype.protocolAction = async function() { + await accessibilityInstance.performScan(browser, commandName); + + return originalProtocolAction.apply(this); + }; + } catch (error) { + Logger.debug(`Failed to patch protocolAction for command ${commandName}`); + } + }); + } else { + commandJson[commandKey].name.forEach(commandName => { + try { + const webElementCommandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const originalCommand = require(webElementCommandPath); + const originalCommandFn = originalCommand.command; + + originalCommand.command = async function(...args) { + await accessibilityInstance.performScan(browser, commandName); + + return originalCommandFn.apply(this, args); + }; + } catch (error) { + Logger.debug(`Failed to patch command ${commandName}`); + } + }); + } + } + } } module.exports = AccessibilityAutomation; diff --git a/src/scripts/accessibilityScripts.js b/src/scripts/accessibilityScripts.js new file mode 100644 index 0000000..a399aa8 --- /dev/null +++ b/src/scripts/accessibilityScripts.js @@ -0,0 +1,107 @@ +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const Logger = require('../utils/logger.js'); + + +class AccessibilityScripts { + static instance = null; + + performScan = null; + getResults = null; + getResultsSummary = null; + saveTestResults = null; + commandsToWrap = null; + ChromeExtension = {}; + + browserstackFolderPath = ''; + commandsPath = ''; + + // don't allow to create instances from it other than through `checkAndGetInstance` + constructor() { + this.browserstackFolderPath = this.getWritableDir(); + this.commandsPath = path.join(this.browserstackFolderPath, 'commands.json'); + } + + static checkAndGetInstance() { + if (!AccessibilityScripts.instance) { + AccessibilityScripts.instance = new AccessibilityScripts(); + AccessibilityScripts.instance.readFromExistingFile(); + } + + return AccessibilityScripts.instance; + } + + getWritableDir() { + const orderedPaths = [ + path.join(os.homedir(), '.browserstack'), + process.cwd(), + os.tmpdir() + ]; + for (const orderedPath of orderedPaths) { + try { + if (fs.existsSync(orderedPath)) { + fs.accessSync(orderedPath); + + return orderedPath; + } + + fs.mkdirSync(orderedPath, {recursive: true}); + + return orderedPath; + + } catch (e) { + Logger.debug(`Failed to access or create directory ${orderedPath}: ${e.message}`); + } + } + + return ''; + } + + readFromExistingFile() { + try { + if (fs.existsSync(this.commandsPath)) { + const data = fs.readFileSync(this.commandsPath, 'utf8'); + if (data) { + this.update(JSON.parse(data)); + } + } + } catch (error) { + Logger.debug(`Failed to read accessibility commands file: ${error.message}`); + } + } + + update(data) { + if (data.scripts) { + this.performScan = data.scripts.scan; + this.getResults = data.scripts.getResults; + this.getResultsSummary = data.scripts.getResultsSummary; + this.saveTestResults = data.scripts.saveResults; + } + if (data.commands && data.commands.length) { + this.commandsToWrap = data.commands; + } + } + + store() { + try { + if (!fs.existsSync(this.browserstackFolderPath)){ + fs.mkdirSync(this.browserstackFolderPath); + } + + fs.writeFileSync(this.commandsPath, JSON.stringify({ + commands: this.commandsToWrap, + scripts: { + scan: this.performScan, + getResults: this.getResults, + getResultsSummary: this.getResultsSummary, + saveResults: this.saveTestResults + } + })); + } catch (error) { + Logger.debug(`Failed to store accessibility commands file: ${error.message}`); + } + } +} + +module.exports = AccessibilityScripts.checkAndGetInstance(); diff --git a/src/testObservability.js b/src/testObservability.js index eb04010..11f78ee 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -9,7 +9,11 @@ const CrashReporter = require('./utils/crashReporter'); const Logger = require('./utils/logger'); const {API_URL, TAKE_SCREENSHOT_REGEX} = require('./utils/constants'); const OrchestrationUtils = require('./testorchestration/orchestrationUtils'); +const AccessibilityAutomation = require('./accessibilityAutomation'); +const accessibilityScripts = require('./scripts/accessibilityScripts'); +const TestMap = require('./utils/testMap'); const hooksMap = {}; +const accessibilityAutomation = new AccessibilityAutomation(); class TestObservability { configure(settings = {}) { @@ -83,36 +87,49 @@ class TestObservability { async launchTestSession() { // Support both old and new configuration options at different levels - const options = this._settings.test_observability || + const testReportingOptions = this._settings.test_observability || this._settings.test_reporting || this._settings.testReportingOptions || this._settings.testObservabilityOptions || this._parentSettings?.testReportingOptions || this._parentSettings?.testObservabilityOptions || {}; + const accessibility = helper.isAccessibilityEnabled(this._parentSettings); + const accessibilityOptions = accessibility ? this._settings.accessibilityOptions || {} : {}; this._gitMetadata = await helper.getGitMetaData(); const fromProduct = { - test_observability: true, - test_reporting: true + test_observability: this._settings.test_observability?.enabled || this._settings.test_reporting?.enabled || false, + accessibility: accessibility }; const data = { format: 'json', project_name: helper.getProjectName(this._settings, this._bstackOptions, fromProduct), name: helper.getBuildName(this._settings, this._bstackOptions, fromProduct), - build_identifier: options.buildIdentifier, - description: options.buildDescription || '', - start_time: new Date().toISOString(), + build_identifier: testReportingOptions.buildIdentifier, + description: testReportingOptions.buildDescription || '', + started_at: new Date().toISOString(), tags: helper.getObservabilityBuildTags(this._settings, this._bstackOptions), host_info: helper.getHostInfo(), ci_info: helper.getCiInfo(), build_run_identifier: process.env.BROWSERSTACK_BUILD_RUN_IDENTIFIER, failed_tests_rerun: process.env.BROWSERSTACK_RERUN || false, version_control: this._gitMetadata, - observability_version: { + accessibility: { + settings: accessibilityOptions + }, + framework_details: { frameworkName: helper.getFrameworkName(this._testRunner), frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion() + sdkVersion: helper.getAgentVersion(), + language: 'javascript', + testFramework: { + name: 'nightwatch', + version: helper.getPackageVersion('nightwatch') + } }, + product_map: this.getProductMapForBuildStartCall(this._parentSettings), + browserstackAutomation: helper.isBrowserstackInfra(this._settings), + config: {}, test_orchestration: this.getTestOrchestrationBuildStartData(this._parentSettings) }; @@ -128,20 +145,18 @@ class TestObservability { }; try { - const response = await makeRequest('POST', 'api/v1/builds', data, config, API_URL); + const response = await makeRequest('POST', 'api/v2/builds', data, config, API_URL); Logger.info('Build creation successful!'); process.env.BS_TESTOPS_BUILD_COMPLETED = true; const responseData = response.data || {}; if (responseData.jwt) { - process.env.BS_TESTOPS_JWT = responseData.jwt; + process.env.BROWSERSTACK_TESTHUB_JWT = responseData.jwt; } if (responseData.build_hashed_id) { - process.env.BS_TESTOPS_BUILD_HASHED_ID = responseData.build_hashed_id; - } - if (responseData.allow_screenshots) { - process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.allow_screenshots.toString(); + process.env.BROWSERSTACK_TESTHUB_UUID = responseData.build_hashed_id; } + this.processLaunchBuildResponse(responseData, this._settings); } catch (error) { if (error.response) { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.response.status} ${error.response.statusText} ${JSON.stringify(error.response.data)}`); @@ -159,11 +174,86 @@ class TestObservability { return orchestrationUtils.getBuildStartData(); } + processLaunchBuildResponse(responseData, settings) { + if (helper.isTestObservabilitySession()) { + this.processTestObservabilityResponse(responseData); + } + this.processAccessibilityResponse(responseData, settings); + } + + processTestObservabilityResponse(responseData) { + if (!responseData.observability) { + this.handleErrorForObservability(null); + + return; + } + if (!responseData.observability.success) { + this.handleErrorForObservability(responseData.observability); + + return; + } + process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'true'; + process.env.BROWSERSTACK_TEST_REPORTING = 'true'; + if (responseData.observability.options.allow_screenshots) { + process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.observability.options.allow_screenshots.toString(); + } + } + + processAccessibilityResponse(responseData, settings) { + if (!responseData.accessibility) { + if (settings.accessibility === true) { + this.handleErrorForAccessibility(null); + } + + return; + } + if (!responseData.accessibility.success) { + this.handleErrorForAccessibility(responseData.accessibility); + + return; + } + + if (responseData.accessibility.options) { + const {accessibilityToken, pollingTimeout, scannerVersion} = helper.jsonifyAccessibilityArray(responseData.accessibility.options.capabilities, 'name', 'value'); + const scriptsJson = { + 'scripts': helper.jsonifyAccessibilityArray(responseData.accessibility.options.scripts, 'name', 'command'), + 'commands': responseData.accessibility.options.commandsToWrap?.commands ?? [] + }; + if (scannerVersion) { + process.env.BSTACK_A11Y_SCANNER_VERSION = scannerVersion; + Logger.debug(`Accessibility scannerVersion ${scannerVersion}`); + } + if (accessibilityToken) { + process.env.BSTACK_A11Y_JWT = accessibilityToken; + process.env.BROWSERSTACK_ACCESSIBILITY = 'true'; + } + if (pollingTimeout) { + process.env.BSTACK_A11Y_POLLING_TIMEOUT = pollingTimeout; + } + if (scriptsJson) { + accessibilityScripts.update(scriptsJson); + accessibilityScripts.store(); + } + } + + } + + handleErrorForObservability(error) { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'false'; + process.env.BROWSERSTACK_TEST_REPORTING = 'false'; + helper.logBuildError(error, 'Test Reporting and Analytics'); + } + + handleErrorForAccessibility(error) { + process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; + helper.logBuildError(error, 'Accessibility'); + } + async stopBuildUpstream () { if (!process.env.BS_TESTOPS_BUILD_COMPLETED) { return; } - if (!process.env.BS_TESTOPS_JWT) { + if (!process.env.BROWSERSTACK_TESTHUB_JWT) { Logger.info('[STOP_BUILD] Missing Authentication Token/ Build ID'); return { @@ -172,11 +262,11 @@ class TestObservability { }; } const data = { - 'stop_time': new Date().toISOString() + 'finished_at': new Date().toISOString() }; const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -184,7 +274,7 @@ class TestObservability { await helper.uploadPending(); await helper.shutDownRequestHandler(); try { - const response = await makeRequest('PUT', `api/v1/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID}/stop`, data, config, API_URL, false); + const response = await makeRequest('PUT', `api/v1/builds/${process.env.BROWSERSTACK_TESTHUB_UUID}/stop`, data, config, API_URL, false); if (response.data?.error) { throw {message: response.data.error}; } else { @@ -207,8 +297,8 @@ class TestObservability { } } - async sendEvents(eventData, testFileReport, startEventType, finishedEventType, hookId, hookType, sectionName) { - await this.sendTestRunEvent(eventData, testFileReport, startEventType, hookId, hookType, sectionName); + async sendHookEvents(eventData, testFileReport, startEventType, finishedEventType, hookId, hookType, sectionName) { + await this.sendHookRunEvent(eventData, testFileReport, startEventType, hookId, hookType, sectionName); if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { @@ -216,7 +306,7 @@ class TestObservability { } } } - await this.sendTestRunEvent(eventData, testFileReport, finishedEventType, hookId, hookType, sectionName); + await this.sendHookRunEvent(eventData, testFileReport, finishedEventType, hookId, hookType, sectionName); } async processTestReportFile(testFileReport) { @@ -232,28 +322,19 @@ class TestObservability { const eventData = completedSections[sectionName]; switch (sectionName) { case '__global_beforeEach_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalBeforeEachHookId, 'GLOBAL_BEFORE_EACH', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalBeforeEachHookId, 'GLOBAL_BEFORE_EACH', sectionName); break; } case '__before_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', beforeHookId, 'BEFORE_ALL', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', beforeHookId, 'BEFORE_ALL', sectionName); break; } case '__after_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', afterHookId, 'AFTER_ALL', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', afterHookId, 'AFTER_ALL', sectionName); break; } case '__global_afterEach_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); - break; - } - default: { - if (eventData.retryTestData?.length>0) { - for (const retryTest of eventData.retryTestData) { - await this.processTestRunData(retryTest, sectionName, testFileReport, hookIds); - } - } - await this.processTestRunData(eventData, sectionName, testFileReport, hookIds); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); break; } } @@ -266,15 +347,11 @@ class TestObservability { } } - async processTestRunData (eventData, sectionName, testFileReport, hookIds) { - const testUuid = uuidv4(); - const errorData = eventData.commands.find(command => command.result?.stack); - eventData.lastError = errorData ? errorData.result : null; - await this.sendTestRunEvent(eventData, testFileReport, 'TestRunStarted', testUuid, null, sectionName, hookIds); + async processTestRunData (eventData, uuid) { if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { - await this.createHttpLogEvent(output, eventData.httpOutput[index + 1], testUuid); + await this.createHttpLogEvent(output, eventData.httpOutput[index + 1], uuid); } } } @@ -291,7 +368,7 @@ class TestObservability { try { if (fs.existsSync(screenshotPath)) { const screenshot = fs.readFileSync(screenshotPath, 'base64'); - await this.createScreenshotLogEvent(testUuid, screenshot, command.startTime); + await this.createScreenshotLogEvent(uuid, screenshot, command.startTime); } } catch (err) { Logger.debug(`Failed to upload screenshot from saveScreenshot: ${err.message}`); @@ -299,11 +376,11 @@ class TestObservability { } else if (TAKE_SCREENSHOT_REGEX.test(command.name) && command.result) { try { if (command.result.value) { - await this.createScreenshotLogEvent(testUuid, command.result.value, command.startTime); + await this.createScreenshotLogEvent(uuid, command.result.value, command.startTime); } else if (command.result.valuePath) { if (fs.existsSync(command.result.valuePath)) { const screenshot = fs.readFileSync(command.result.valuePath, 'utf8'); - await this.createScreenshotLogEvent(testUuid, screenshot, command.startTime); + await this.createScreenshotLogEvent(uuid, screenshot, command.startTime); } } } catch (err) { @@ -312,9 +389,8 @@ class TestObservability { } } } - await this.sendTestRunEvent(eventData, testFileReport, 'TestRunFinished', testUuid, null, sectionName, hookIds); } - + async sendSkippedTestEvent(skippedTest, testFileReport) { const testData = { uuid: uuidv4(), @@ -384,7 +460,76 @@ class TestObservability { } } - async sendTestRunEvent(eventData, testFileReport, eventType, uuid, hookType, sectionName, hooks) { + async sendTestRunEvent(eventType, test, uuid) { + Logger.debug(`Sending test run event with eventType: ${eventType}`); + const testMetaData = test.metadata; + const testName = test.testcase; + const settings = test.settings || {}; + const startTimestamp = test.envelope[testName].startTimestamp; + const testResults = {}; + const testBody = this.getTestBody(test.testCaseData); + const provider = helper.getCloudProvider(testMetaData.host); + const testData = { + uuid: uuid, + type: 'test', + name: testName, + body: { + lang: 'javascript', + code: testBody ? testBody.toString() : null + }, + scope: `${testMetaData.name} - ${testName}`, + scopes: [ + testMetaData.name + ], + tags: testMetaData.tags, + identifier: `${testMetaData.name} - ${testName}`, + file_name: path.relative(process.cwd(), testMetaData.modulePath), + location: path.relative(process.cwd(), testMetaData.modulePath), + vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testMetaData.modulePath) : null, + started_at: new Date(startTimestamp).toISOString(), + result: 'pending', + framework: 'nightwatch', + integrations: { + [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) + }, + product_map: { + observability: helper.isTestObservabilitySession(), + accessibility: helper.isAccessibilitySession() && accessibilityAutomation.shouldScanTestForAccessibility(test) && process.env.VALID_ALLY_PLATFORM + } + }; + + if (eventType === 'TestRunFinished') { + const eventData = test.envelope[testName].testcase; + testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(startTimestamp).toISOString(); + testData.result = 'passed'; + if (eventData && eventData.commands && Array.isArray(eventData.commands)) { + const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail'); + if (failedCommand) { + testData.result = 'failed'; + if (failedCommand.result) { + testData.failure = [ + { + 'backtrace': [stripAnsi(failedCommand.result.message || ''), failedCommand.result.stack || ''] + } + ]; + testData.failure_reason = failedCommand.result.message ? stripAnsi(failedCommand.result.message) : null; + if (failedCommand.result.name) { + testData.failure_type = failedCommand.result.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + } + } + } + } + await this.processTestRunData (eventData, uuid); + } + + const uploadData = { + event_type: eventType, + test_run: testData + }; + await helper.uploadEventData(uploadData); + } + + async sendHookRunEvent(eventData, testFileReport, eventType, uuid, hookType, sectionName, hooks) { const testData = { uuid: uuid, type: 'hook', @@ -404,12 +549,12 @@ class TestObservability { hook_type: hookType }; - if (eventType === 'HookRunFinished' || eventType === 'TestRunFinished') { + if (eventType === 'HookRunFinished') { testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(eventData.startTimestamp).toISOString(); testData.result = eventData.status === 'pass' ? 'passed' : 'failed'; testData.duration_in_ms = 'timeMs' in eventData ? eventData.timeMs : eventData.time; if (eventData.status === 'fail' && eventData.lastError) { - testData.failure = [ + testData.failure_data = [ { 'backtrace': [stripAnsi(eventData.lastError.message), eventData.lastError.stack] } @@ -420,7 +565,7 @@ class TestObservability { } } else if (eventData.status === 'fail' && (testFileReport?.completed[sectionName]?.lastError || testFileReport?.completed[sectionName]?.stackTrace)) { const testCompletionData = testFileReport.completed[sectionName]; - testData.failure = [ + testData.failure_data = [ {'backtrace': [testCompletionData?.stackTrace]} ]; testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; @@ -434,26 +579,10 @@ class TestObservability { testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); } - if (eventType === 'TestRunStarted') { - testData.type = 'test'; - testData.integrations = {}; - const provider = helper.getCloudProvider(testFileReport.host); - testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); - } - - if (eventType === 'TestRunFinished') { - testData.type = 'test'; - testData.hooks = hooks; - } - const uploadData = { - event_type: eventType + event_type: eventType, + hook_run: testData }; - if (eventType.match(/HookRun/)) { - uploadData['hook_run'] = testData; - } else { - uploadData['test_run'] = testData; - } await helper.uploadEventData(uploadData); } @@ -709,6 +838,25 @@ class TestObservability { Logger.error(`Exception in uploading log data to Test Reporting and Analytics with error : ${error}`); } } -} + + getProductMapForBuildStartCall(settings) { + const product = helper.getObservabilityLinkedProductName(settings.desiredCapabilities, settings?.selenium?.host); + + const buildProductMap = { + automate: product === 'automate', + app_automate: product === 'app-automate', + observability: helper.isTestObservabilitySession(), + accessibility: helper.isAccessibilityEnabled(settings), + turboscale: product === 'turboscale', + percy: false + }; + + return buildProductMap; + } + + getTestBody(testCaseData) { + return testCaseData?.context.__module[testCaseData.testName] || null; + } +} module.exports = TestObservability; diff --git a/src/testorchestration/orchestrationUtils.js b/src/testorchestration/orchestrationUtils.js index c9965f6..63787f3 100644 --- a/src/testorchestration/orchestrationUtils.js +++ b/src/testorchestration/orchestrationUtils.js @@ -145,7 +145,7 @@ class OrchestrationUtils { * Check if the abort build file exists */ static checkAbortBuildFileExists() { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; const filePath = path.join(tmpdir(), `abort_build_${buildUuid}`); return fs.existsSync(filePath); @@ -155,7 +155,7 @@ class OrchestrationUtils { * Write failure to file */ static writeFailureToFile(testName) { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; const failedTestsFile = path.join(tmpdir(), `failed_tests_${buildUuid}.txt`); fs.appendFileSync(failedTestsFile, `${testName}\n`); @@ -433,7 +433,7 @@ class OrchestrationUtils { * Collects build data by making a call to the collect-build-data endpoint */ async collectBuildData(config) { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; this.logger.debug(`[collectBuildData] Collecting build data for build UUID: ${buildUuid}`); try { diff --git a/src/testorchestration/requestUtils.js b/src/testorchestration/requestUtils.js index 254dc9e..ef7e2d0 100644 --- a/src/testorchestration/requestUtils.js +++ b/src/testorchestration/requestUtils.js @@ -36,11 +36,11 @@ class RequestUtils { * Makes an orchestration request with the given method and data */ static async makeOrchestrationRequest(method, reqEndpoint, options) { - const jwtToken = process.env.BS_TESTOPS_JWT || ''; + const jwtToken = process.env.BROWSERSTACK_TESTHUB_JWT || ''; // Validate JWT token if (!jwtToken) { - Logger.error('BS_TESTOPS_JWT environment variable is not set. This is required for test orchestration.'); + Logger.error('BROWSERSTACK_TESTHUB_JWT environment variable is not set. This is required for test orchestration.'); return null; } diff --git a/src/utils/crashReporter.js b/src/utils/crashReporter.js index 8caa23f..1255def 100644 --- a/src/utils/crashReporter.js +++ b/src/utils/crashReporter.js @@ -52,7 +52,7 @@ class CrashReporter { try { const data = { - hashed_id: process.env.BS_TESTOPS_BUILD_HASHED_ID, + hashed_id: process.env.BROWSERSTACK_TESTHUB_UUID, observability_version: { frameworkName: 'nightwatch-default', frameworkVersion: helper.getPackageVersion('nightwatch'), diff --git a/src/utils/helper.js b/src/utils/helper.js index 1054240..ba103d6 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -72,10 +72,42 @@ exports.getObservabilityKey = (config, bstackOptions={}) => { return process.env.BROWSERSTACK_ACCESS_KEY || config?.key || bstackOptions?.accessKey; }; +exports.isAppAutomate = () => { + return process.env.BROWSERSTACK_APP_AUTOMATE === 'true'; +}; + +exports.checkTestEnvironmentForAppAutomate = (testEnvSettings) => { + + const firstEnvKey = Object.keys(testEnvSettings)[0]; + const firstEnv = testEnvSettings[firstEnvKey]; + if (firstEnv?.desiredCapabilities?.['appium:options']?.app) { + return true; + } + + return false; +}; + exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; +exports.isTestHubBuild = (pluginSettings = {}, isBuildStart = false) => { + if (isBuildStart) { + return pluginSettings?.test_reporting?.enabled === true || pluginSettings?.test_observability?.enabled === true || pluginSettings?.accessibility === true; + } + + return this.isTestObservabilitySession() || this.isAccessibilitySession(); + +}; + +exports.isAccessibilityEnabled = (settings) => { + if (process.argv.includes('--disable-accessibility')) {return false} + + if (process.env.BROWSERSTACK_ACCESSIBILITY === 'false') {return false} + + return settings['@nightwatch/browserstack']?.accessibility === true; +}; + exports.getProjectName = (options, bstackOptions={}, fromProduct={}) => { if ((fromProduct.test_observability || fromProduct.test_reporting) && ((options.test_observability && options.test_observability.projectName) || @@ -376,8 +408,10 @@ exports.getHostInfo = () => { }; }; -exports.isBrowserstackInfra = () => { - return process.env.BROWSERSTACK_INFRA === 'true'; +exports.isBrowserstackInfra = (settings) => { + const isBrowserstackInfra = settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1 ? false : true; + + return isBrowserstackInfra; }; const findGitConfig = async (filePath) => { @@ -505,12 +539,12 @@ exports.uploadEventData = async (eventData) => { ['HookRunFinished']: 'Hook_End_Upload' }[eventData.event_type]; - if (process.env.BS_TESTOPS_JWT && process.env.BS_TESTOPS_JWT !== 'null') { + if (process.env.BROWSERSTACK_TESTHUB_JWT && process.env.BROWSERSTACK_TESTHUB_JWT !== 'null') { requestQueueHandler.pending_test_uploads += 1; } if (process.env.BS_TESTOPS_BUILD_COMPLETED === 'true') { - if (process.env.BS_TESTOPS_JWT === 'null') { + if (process.env.BROWSERSTACK_TESTHUB_JWT === 'null') { Logger.info(`EXCEPTION IN ${log_tag} REQUEST TO TEST REPORTING AND ANALYTICS : missing authentication token`); requestQueueHandler.pending_test_uploads = Math.max(0, requestQueueHandler.pending_test_uploads-1); @@ -537,7 +571,7 @@ exports.uploadEventData = async (eventData) => { const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -618,7 +652,7 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { if (hostname) { if (hostname.includes('browserstack.com') && !hostname.includes('hub-ft')) { - if (this.isUndefined(caps.browserName)) { + if (this.isAppAutomate()) { product = 'app-automate'; } else { product = 'automate'; @@ -631,13 +665,14 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { return product; }; -exports.getIntegrationsObject = (capabilities, sessionId, hostname) => { +exports.getIntegrationsObject = (capabilities, sessionId, hostname, platform_version) => { return { capabilities: capabilities, session_id: sessionId, browser: capabilities.browserName, browser_version: capabilities.browserVersion, platform: capabilities.platformName, + platform_version: capabilities.platformVersion || platform_version, product: this.getObservabilityLinkedProductName(capabilities, hostname) }; }; @@ -1301,4 +1336,67 @@ exports.getGitMetadataForAiSelection = (folders = []) => { })); return formattedResults; -}; \ No newline at end of file +}; + +exports.jsonifyAccessibilityArray = (dataArray, keyName, valueName) => { + const result = {}; + dataArray.forEach((element) => { + result[element[keyName]] = element[valueName]; + }); + + return result; +}; + +exports.logBuildError = (error, product = '') => { + if (!error || !error.errors) { + Logger.error(`${product.toUpperCase()} Build creation failed ${error}`); + + return; + } + + for (const errorJson of error.errors) { + const errorType = errorJson.key; + const errorMessage = errorJson.message; + if (errorMessage) { + switch (errorType) { + case 'ERROR_INVALID_CREDENTIALS': + Logger.error(errorMessage); + break; + case 'ERROR_ACCESS_DENIED': + Logger.info(errorMessage); + break; + case 'ERROR_SDK_DEPRECATED': + Logger.error(errorMessage); + break; + default: + Logger.error(errorMessage); + } + } + } +}; + +exports.patchBrowserTerminateCommand = () =>{ + + const nightwatchDir = path.dirname(require.resolve('nightwatch')); + const CommandPath = path.join(nightwatchDir, 'testsuite/index.js'); + const TestSuite = require(CommandPath); + const originalFn = TestSuite.prototype.terminate; + TestSuite.prototype.terminate = async function(...args) { + const maxWaitTime = 30000; + const pollInterval = 500; + const startTime = Date.now(); + const AccessibilityAutomation = require('../accessibilityAutomation'); + while (Date.now() - startTime < maxWaitTime) { + const pendingAllyReq = AccessibilityAutomation.pendingAllyReq || 0; + + if (pendingAllyReq === 0) { + break; + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } + Logger.debug(`Pending Accessibility requests at session end: ${AccessibilityAutomation.pendingAllyReq }`); + + return originalFn.apply(this, args); + }; +}; + diff --git a/src/utils/logPatcher.js b/src/utils/logPatcher.js index 9cd08a6..e0ecd1e 100644 --- a/src/utils/logPatcher.js +++ b/src/utils/logPatcher.js @@ -68,7 +68,7 @@ class LogPatcher extends Transport { process.on('message', (data) => { if (data.uuid !== undefined){ _uuid = data.uuid; - process.env.TEST_OPS_TEST_UUID = _uuid; + process.env.TEST_RUN_UUID = _uuid; } }); process.on('disconnect', async () => { diff --git a/src/utils/requestQueueHandler.js b/src/utils/requestQueueHandler.js index e4363ec..f9ad180 100644 --- a/src/utils/requestQueueHandler.js +++ b/src/utils/requestQueueHandler.js @@ -93,7 +93,7 @@ class RequestQueueHandler { async batchAndPostEvents (eventUrl, kind, data) { const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -101,7 +101,7 @@ class RequestQueueHandler { try { const response = await makeRequest('POST', eventUrl, data, config); - if (response.data.error) { + if (response.data && response.data.error) { throw ({message: response.data.error}); } else { this.pending_test_uploads = Math.max(0, this.pending_test_uploads - data.length); diff --git a/src/utils/testMap.js b/src/utils/testMap.js new file mode 100644 index 0000000..39b774c --- /dev/null +++ b/src/utils/testMap.js @@ -0,0 +1,95 @@ +const {v4: uuidv4} = require('uuid'); + +const sharedTestMap = new Map(); +let sharedCurrentTest = null; +const activeTestRuns = new Map(); + +class TestMap { + + static storeTestDetails(test) { + const testIdentifier = this.generateTestIdentifier(test); + const uuid = this.generateUUID(); + + if (!sharedTestMap.has(testIdentifier)) { + sharedTestMap.set(testIdentifier, { + baseUuid: uuid, // Store the first UUID as base + retries: [], + currentUuid: uuid, + test, + createdAt: new Date().toISOString() + }); + } else { + // This is a retry - add new UUID to retries array + const testData = sharedTestMap.get(testIdentifier); + testData.retries.push({ + uuid, + startedAt: new Date().toISOString() + }); + testData.currentUuid = uuid; // Update to latest UUID + sharedTestMap.set(testIdentifier, testData); + } + + // Track this as an active test run + activeTestRuns.set(uuid, { + identifier: testIdentifier, + startedAt: new Date().toISOString(), + hasFinished: false + }); + + sharedCurrentTest = testIdentifier; + + return uuid; + } + + static getUUID(test = null) { + if (test) { + const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); + const testData = sharedTestMap.get(testIdentifier); + + return testData ? testData.currentUuid : null; + } + + return null; + } + + static markTestFinished(uuid) { + if (activeTestRuns.has(uuid)) { + const testRun = activeTestRuns.get(uuid); + testRun.hasFinished = true; + testRun.finishedAt = new Date().toISOString(); + activeTestRuns.set(uuid, testRun); + + return true; + } + + return false; + } + + static hasTestFinished(uuid) { + const testRun = activeTestRuns.get(uuid); + + return testRun ? testRun.hasFinished : false; + } + + + static getTestDetails(identifier) { + return sharedTestMap.has(identifier) ? sharedTestMap.get(identifier) : null; + } + + static generateTestIdentifier(test) { + if (!test) { + throw new Error('Test object is required to generate identifier'); + } + const testName = test.testcase; + const moduleName = test.metadata.name; + + return `${moduleName}::${testName}`; + } + + static generateUUID() { + return uuidv4(); + } +} + +module.exports = TestMap; + diff --git a/test/src/test-observability/processTestRunData.js b/test/src/test-observability/processTestRunData.js index 4a4c6aa..c5c46aa 100644 --- a/test/src/test-observability/processTestRunData.js +++ b/test/src/test-observability/processTestRunData.js @@ -8,37 +8,28 @@ describe('TestObservability - processTestRunData', function () { this.sandbox = sinon.createSandbox(); this.testObservability = new TestObservability(); - this.eventData = {commands: []}; - this.sectionName = 'testSection'; - this.testFileReport = {}; - this.hookIds = []; - this.sendTestRunEventStub = this.sandbox.stub(this.testObservability, 'sendTestRunEvent').resolves(); + this.eventData = {commands: [], httpOutput: []}; + this.uuid = 'test-uuid-123'; }); afterEach(() => { this.sandbox.restore(); }); - it('should send test run events', async () => { - await this.testObservability.processTestRunData(this.eventData, this.sectionName, this.testFileReport, this.hookIds); - sinon.assert.calledTwice(this.sendTestRunEventStub); - sinon.assert.calledWith(this.sendTestRunEventStub.firstCall, this.eventData, this.testFileReport, 'TestRunStarted', sinon.match.string, null, this.sectionName, this.hookIds); - sinon.assert.calledWith(this.sendTestRunEventStub.secondCall, this.eventData, this.testFileReport, 'TestRunFinished', sinon.match.string, null, this.sectionName, this.hookIds); - }); - it('should create screenshot log events', async () => { this.eventData = { commands: [ {name: 'saveScreenshot', args: ['path/to/screenshot.png'], startTime: 'start_time'} - ] + ], + httpOutput: [] }; this.sandbox.stub(fs, 'existsSync').callsFake(() => true); this.sandbox.stub(fs, 'readFileSync').callsFake(() => 'screenshot-base-64'); const createScreenshotLogEventStub = this.sandbox.stub(this.testObservability, 'createScreenshotLogEvent'); process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'true'; - await this.testObservability.processTestRunData(this.eventData, this.sectionName, this.testFileReport, this.hookIds); + await this.testObservability.processTestRunData(this.eventData, this.uuid); process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'false'; - sinon.assert.calledOnceWithExactly(createScreenshotLogEventStub, sinon.match.string, 'screenshot-base-64', 'start_time'); + sinon.assert.calledOnceWithExactly(createScreenshotLogEventStub, this.uuid, 'screenshot-base-64', 'start_time'); }); }); diff --git a/test/src/utils/helper.js b/test/src/utils/helper.js index c82372f..2d54cb7 100644 --- a/test/src/utils/helper.js +++ b/test/src/utils/helper.js @@ -326,21 +326,30 @@ describe('isBrowserstackInfra', () => { isBrowserstackInfra = require('../../../src/utils/helper').isBrowserstackInfra; }); - it('returns false for undefined', async () => { - delete process.env.BROWSERSTACK_INFRA; - expect(isBrowserstackInfra()).to.be.false; + it('returns true for undefined settings', async () => { + expect(isBrowserstackInfra()).to.be.true; }); - it('returns true if env variable is set to true', async () => { - process.env.BROWSERSTACK_INFRA = true; - expect(isBrowserstackInfra()).to.be.true; - delete process.env.BROWSERSTACK_INFRA; + it('returns true for empty settings', async () => { + expect(isBrowserstackInfra({})).to.be.true; }); - it('returns false if env variable is set to false', async () => { - process.env.BROWSERSTACK_INFRA = false; - expect(isBrowserstackInfra()).to.be.false; - delete process.env.BROWSERSTACK_INFRA; + it('returns true if webdriver.host contains browserstack', async () => { + const settings = { + webdriver: { + host: 'hub-cloud.browserstack.com' + } + }; + expect(isBrowserstackInfra(settings)).to.be.true; + }); + + it('returns false if webdriver.host does not contain browserstack', async () => { + const settings = { + webdriver: { + host: 'localhost' + } + }; + expect(isBrowserstackInfra(settings)).to.be.false; }); }); From 532a7c1657202d6e5243641a9e92bbbc9242730f Mon Sep 17 00:00:00 2001 From: Priyansh Garg Date: Tue, 9 Dec 2025 19:38:24 +0530 Subject: [PATCH 09/12] npm audit fix --- package-lock.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4f9a492..4e09698 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1584,10 +1584,11 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, + "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, @@ -3947,9 +3948,9 @@ "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" }, "js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "dev": true, "requires": { "argparse": "^2.0.1" From 3c723ab8c19c12e3e86ce01e109ab90106479578 Mon Sep 17 00:00:00 2001 From: Priyansh Garg Date: Tue, 9 Dec 2025 19:40:42 +0530 Subject: [PATCH 10/12] 3.7.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4e09698..f9bcb48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@nightwatch/browserstack", - "version": "3.6.2", + "version": "3.7.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@nightwatch/browserstack", - "version": "3.6.2", + "version": "3.7.0", "license": "MIT", "dependencies": { "@cypress/request": "^3.0.1", diff --git a/package.json b/package.json index 8277a3d..3832a86 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@nightwatch/browserstack", - "version": "3.6.2", + "version": "3.7.0", "description": "Nightwatch plugin for integration with browserstack.", "main": "index.js", "scripts": { From b591245e417da79417de7d3a4520942e7b3678f1 Mon Sep 17 00:00:00 2001 From: harshit-bstack Date: Wed, 10 Dec 2025 17:43:46 +0530 Subject: [PATCH 11/12] feat: Add glob dependency and refactor glob pattern matching in helper functions (#45) * feat: Add glob dependency and refactor glob pattern matching in helper functions * Glob Version update * Eslint Fix --- package.json | 3 +- src/utils/helper.js | 119 +++++--------------------------------------- 2 files changed, 14 insertions(+), 108 deletions(-) diff --git a/package.json b/package.json index 3832a86..c27d1e1 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "@cypress/request": "^3.0.1", "strip-ansi": "^6.0.1", "winston-transport": "^4.5.0", - "uuid": "^9.0.0" + "uuid": "^9.0.0", + "glob": "^7.2.0" }, "devDependencies": { "chai": "^4.3.7", diff --git a/src/utils/helper.js b/src/utils/helper.js index ba103d6..c7eb798 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -2,6 +2,7 @@ const os = require('os'); const fs = require('fs'); const fsPromises = fs.promises; const path = require('path'); +const glob = require('glob'); const {promisify} = require('util'); const gitRepoInfo = require('git-repo-info'); const gitconfig = require('gitconfiglocal'); @@ -888,84 +889,7 @@ exports.truncateString = (field, truncateSizeInBytes) => { // Helper function to check if a pattern contains glob characters exports.isGlobPattern = (pattern) => { - return pattern.includes('*') || pattern.includes('?') || pattern.includes('['); -}; - -// Helper function to recursively find files matching a pattern -exports.findFilesRecursively = (dir, pattern) => { - const files = []; - try { - if (!fs.existsSync(dir)) { - return files; - } - - const entries = fs.readdirSync(dir, {withFileTypes: true}); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - - if (entry.isDirectory()) { - // Recursively search subdirectories - files.push(...exports.findFilesRecursively(fullPath, pattern)); - } else if (entry.isFile()) { - const relativePath = path.relative(process.cwd(), fullPath); - - // Enhanced pattern matching for glob patterns - if (exports.matchesGlobPattern(relativePath, pattern)) { - files.push(relativePath); - } - } - } - } catch (err) { - Logger.debug(`Error reading directory ${dir}: ${err.message}`); - } - - return files; -}; - -// Helper function to match a file path against a glob pattern -exports.matchesGlobPattern = (filePath, pattern) => { - // Normalize paths to use forward slashes - const normalizedPath = filePath.replace(/\\/g, '/'); - const normalizedPattern = pattern.replace(/\\/g, '/'); - - // Convert glob pattern to regex step by step - let regexPattern = normalizedPattern; - - // First, handle ** patterns (must be done before single *) - // ** should match zero or more directories - regexPattern = regexPattern.replace(/\*\*/g, '§DOUBLESTAR§'); - - // Escape regex special characters except the placeholders - regexPattern = regexPattern.replace(/[.+^${}()|[\]\\]/g, '\\$&'); - - // Now handle single * and ? patterns - regexPattern = regexPattern.replace(/\*/g, '[^/]*'); // * matches anything except path separators - regexPattern = regexPattern.replace(/\?/g, '[^/]'); // ? matches single character except path separator - - // Finally, replace ** placeholder with regex for any path (including zero directories) - regexPattern = regexPattern.replace(/§DOUBLESTAR§/g, '.*?'); - - // Special case: if pattern ends with /**/* we need to handle direct files in the base directory - // Convert patterns like "dir/**/*" to also match "dir/*" - if (normalizedPattern.includes('/**/')) { - const baseRegex = regexPattern; - const alternativeRegex = regexPattern.replace(/\/\.\*\?\//g, '/'); - regexPattern = `(?:${baseRegex}|${alternativeRegex})`; - } - - // Ensure pattern matches from start to end - regexPattern = '^' + regexPattern + '$'; - - try { - const regex = new RegExp(regexPattern); - - return regex.test(normalizedPath); - } catch (err) { - Logger.debug(`Error in glob pattern matching: ${err.message}`); - - return false; - } + return glob.hasMagic(pattern); }; // Helper function to resolve and collect test files from a path/pattern @@ -1039,36 +963,17 @@ exports.findTestFilesInDirectory = (dir) => { exports.expandGlobPattern = (pattern) => { Logger.debug(`Expanding glob pattern: ${pattern}`); - // Extract the base directory from the pattern - const parts = pattern.split(/[/\\]/); - let baseDir = '.'; - let patternStart = 0; - - // Find the first part that contains glob characters - for (let i = 0; i < parts.length; i++) { - if (exports.isGlobPattern(parts[i])) { - patternStart = i; - break; - } - if (i === 0 && parts[i] !== '.') { - baseDir = parts[i]; - } else if (i > 0) { - baseDir = path.join(baseDir, parts[i]); - } - } - - // If baseDir doesn't exist, try current directory - if (!fs.existsSync(baseDir)) { - Logger.debug(`Base directory ${baseDir} doesn't exist, using current directory`); - baseDir = '.'; + try { + const files = glob.sync(pattern); + + Logger.debug(`Found ${files.length} files matching pattern: ${pattern}`); + + return files; + } catch (err) { + Logger.debug(`Error expanding glob pattern: ${err.message}`); + + return []; } - - Logger.debug(`Base directory: ${baseDir}, Pattern: ${pattern}`); - - const files = exports.findFilesRecursively(baseDir, pattern); - Logger.debug(`Found ${files.length} files matching pattern: ${pattern}`); - - return files; }; /** From 680b8ce11ee996b3b30a3221c04f547d0b8e6359 Mon Sep 17 00:00:00 2001 From: Priyansh Garg Date: Wed, 10 Dec 2025 17:55:34 +0530 Subject: [PATCH 12/12] 3.7.1 --- package-lock.json | 5 +++-- package.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f9bcb48..36c8b89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@nightwatch/browserstack", - "version": "3.7.0", + "version": "3.7.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@nightwatch/browserstack", - "version": "3.7.0", + "version": "3.7.1", "license": "MIT", "dependencies": { "@cypress/request": "^3.0.1", @@ -14,6 +14,7 @@ "git-last-commit": "^1.0.1", "git-repo-info": "^2.1.1", "gitconfiglocal": "^2.1.0", + "glob": "^7.2.0", "strip-ansi": "^6.0.1", "uuid": "^9.0.0", "winston-transport": "^4.5.0" diff --git a/package.json b/package.json index c27d1e1..67f8894 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@nightwatch/browserstack", - "version": "3.7.0", + "version": "3.7.1", "description": "Nightwatch plugin for integration with browserstack.", "main": "index.js", "scripts": {