From 0b01154283e0d024b02de29bc6792c0bb6e5037a Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Thu, 18 Dec 2025 11:49:17 -0800 Subject: [PATCH 1/9] feat: Implement Heretto CMS uploader for automatic screenshot synchronization - Added HerettoUploader class to handle file uploads to Heretto CMS. - Integrated uploader into the existing upload process for changed screenshots. - Enhanced screenshot saving logic to track changes and source integration metadata. - Created documentation for Heretto screenshot sync feature, detailing the upload process and architecture. - Updated resolver to build file mappings and handle Heretto-specific configurations. - Introduced utility functions to find Heretto integrations based on file paths. - Added error handling and logging for upload operations to improve reliability. --- package-lock.json | 33 +++- package.json | 1 + src/heretto.js | 373 ++++++++++++++++++++++++++++++++++++++++++++++ src/utils.js | 66 ++++++++ 4 files changed, 471 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1b40437..abff311 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "axios": "^1.13.2", "doc-detective-common": "3.6.0-dev.1", "dotenv": "^17.2.3", + "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", "posthog-node": "^5.17.2" }, @@ -232,7 +233,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -953,6 +953,24 @@ "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" }, + "node_modules/fast-xml-parser": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.3.tgz", + "integrity": "sha512-2O3dkPAAC6JavuMm8+4+pgTk+5hoAs+CjZ+sWcQLkX9+/tHRuTkQh/Oaifr8qDmZ8iEHb771Ea6G8CdwkrgvYA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fill-keys": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", @@ -1383,7 +1401,6 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", - "peer": true, "engines": { "node": ">= 10.16.0" } @@ -2339,6 +2356,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", + "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", diff --git a/package.json b/package.json index b6e7b26..f74d3f3 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "axios": "^1.13.2", "doc-detective-common": "3.6.0-dev.1", "dotenv": "^17.2.3", + "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", "posthog-node": "^5.17.2" }, diff --git a/src/heretto.js b/src/heretto.js index c535e33..968b8bb 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -4,6 +4,7 @@ const path = require("path"); const os = require("os"); const crypto = require("crypto"); const AdmZip = require("adm-zip"); +const { XMLParser } = require("fast-xml-parser"); // Internal constants - not exposed to users const POLLING_INTERVAL_MS = 5000; @@ -11,6 +12,8 @@ const POLLING_TIMEOUT_MS = 300000; // 5 minutes const API_REQUEST_TIMEOUT_MS = 30000; // 30 seconds for individual API requests const DOWNLOAD_TIMEOUT_MS = 300000; // 5 minutes for downloads const DEFAULT_SCENARIO_NAME = "Doc Detective"; +// Base URL for REST API (different from publishing API) +const REST_API_PATH = "/rest/all-files"; /** * Creates a Base64-encoded Basic Auth header from username and API token. @@ -52,6 +55,25 @@ function createApiClient(herettoConfig) { }); } +/** + * Creates an axios instance configured for Heretto REST API requests (different base URL). + * @param {Object} herettoConfig - Heretto integration configuration + * @returns {Object} Configured axios instance for REST API + */ +function createRestApiClient(herettoConfig) { + const authHeader = createAuthHeader( + herettoConfig.username, + herettoConfig.apiToken + ); + return axios.create({ + baseURL: `https://${herettoConfig.organizationId}.heretto.com`, + timeout: API_REQUEST_TIMEOUT_MS, + headers: { + Authorization: `Basic ${authHeader}`, + }, + }); +} + /** * Fetches all available publishing scenarios from Heretto. * @param {Object} client - Configured axios instance @@ -404,6 +426,17 @@ async function loadHerettoContent(herettoConfig, log, config) { config ); + // Build file mapping from extracted content + if (outputPath && herettoConfig.uploadOnChange) { + const fileMapping = await buildFileMapping( + outputPath, + herettoConfig, + log, + config + ); + herettoConfig.fileMapping = fileMapping; + } + return outputPath; } catch (error) { log( @@ -415,14 +448,354 @@ async function loadHerettoContent(herettoConfig, log, config) { } } +/** + * Builds a mapping of local file paths to Heretto file metadata. + * Parses DITA files to extract file references and attempts to resolve UUIDs. + * @param {string} outputPath - Path to extracted Heretto content + * @param {Object} herettoConfig - Heretto integration configuration + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Mapping of local paths to {fileId, filePath} + */ +async function buildFileMapping(outputPath, herettoConfig, log, config) { + const fileMapping = {}; + const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + }); + + try { + // Recursively find all DITA/XML files + const ditaFiles = findFilesWithExtensions(outputPath, [ + ".dita", + ".ditamap", + ".xml", + ]); + + for (const ditaFile of ditaFiles) { + try { + const content = fs.readFileSync(ditaFile, "utf-8"); + const parsed = xmlParser.parse(content); + + // Extract image references from DITA content + const imageRefs = extractImageReferences(parsed); + + for (const imageRef of imageRefs) { + // Resolve relative path to absolute local path + const absoluteLocalPath = path.resolve( + path.dirname(ditaFile), + imageRef + ); + + if (!fileMapping[absoluteLocalPath]) { + fileMapping[absoluteLocalPath] = { + filePath: imageRef, + sourceFile: ditaFile, + }; + } + } + } catch (parseError) { + log( + config, + "debug", + `Failed to parse ${ditaFile} for file mapping: ${parseError.message}` + ); + } + } + + log( + config, + "debug", + `Built file mapping with ${Object.keys(fileMapping).length} entries` + ); + } catch (error) { + log(config, "warning", `Failed to build file mapping: ${error.message}`); + } + + return fileMapping; +} + +/** + * Recursively finds files with specified extensions. + * @param {string} dir - Directory to search + * @param {Array} extensions - File extensions to match (e.g., ['.dita', '.xml']) + * @returns {Array} Array of matching file paths + */ +function findFilesWithExtensions(dir, extensions) { + const results = []; + + try { + const items = fs.readdirSync(dir); + + for (const item of items) { + const fullPath = path.join(dir, item); + const stat = fs.statSync(fullPath); + + if (stat.isDirectory()) { + results.push(...findFilesWithExtensions(fullPath, extensions)); + } else if ( + extensions.some((ext) => fullPath.toLowerCase().endsWith(ext)) + ) { + results.push(fullPath); + } + } + } catch (error) { + // Ignore read errors for inaccessible directories + } + + return results; +} + +/** + * Extracts image references from parsed DITA XML content. + * Looks for elements with href attributes. + * @param {Object} parsedXml - Parsed XML object + * @returns {Array} Array of image href values + */ +function extractImageReferences(parsedXml) { + const refs = []; + + function traverse(obj) { + if (!obj || typeof obj !== "object") return; + + // Check for image elements + if (obj.image) { + const images = Array.isArray(obj.image) ? obj.image : [obj.image]; + for (const img of images) { + if (img["@_href"]) { + refs.push(img["@_href"]); + } + } + } + + // Recursively traverse all properties + for (const key of Object.keys(obj)) { + if (typeof obj[key] === "object") { + traverse(obj[key]); + } + } + } + + traverse(parsedXml); + return refs; +} + +/** + * Searches for a file in Heretto by filename. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {string} filename - Name of the file to search for + * @param {string} folderPath - Optional folder path to search within + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} File info with ID and URI, or null if not found + */ +async function searchFileByName( + herettoConfig, + filename, + folderPath, + log, + config +) { + const client = createApiClient(herettoConfig); + + try { + const searchBody = { + queryString: filename, + foldersToSearch: {}, + startOffset: 0, + endOffset: 10, + searchResultType: "FILES_ONLY", + addPrefixAndFuzzy: false, + }; + + // If folderPath provided, search within that folder; otherwise search root + if (folderPath) { + searchBody.foldersToSearch[folderPath] = true; + } else { + // Search in organization root + searchBody.foldersToSearch[ + `/db/organizations/${herettoConfig.organizationId}/` + ] = true; + } + + const response = await client.post( + "/ezdnxtgen/api/search".replace("/ezdnxtgen/api/v2", ""), + searchBody, + { + baseURL: `https://${herettoConfig.organizationId}.heretto.com`, + headers: { "Content-Type": "application/json" }, + } + ); + + if (response.data?.hits?.length > 0) { + // Find exact filename match + const exactMatch = response.data.hits.find( + (hit) => hit.fileEntity?.name === filename + ); + + if (exactMatch) { + return { + fileId: exactMatch.fileEntity.ID, + filePath: exactMatch.fileEntity.URI, + name: exactMatch.fileEntity.name, + }; + } + } + + return null; + } catch (error) { + log( + config, + "debug", + `Failed to search for file "${filename}": ${error.message}` + ); + return null; + } +} + +/** + * Uploads a file to Heretto CMS. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {string} fileId - UUID of the file to update + * @param {string} localFilePath - Local path to the file to upload + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Result object with status and description + */ +async function uploadFile(herettoConfig, fileId, localFilePath, log, config) { + const client = createRestApiClient(herettoConfig); + + try { + // Read file as binary + const fileBuffer = fs.readFileSync(localFilePath); + + // Determine content type from file extension + const ext = path.extname(localFilePath).toLowerCase(); + let contentType = "application/octet-stream"; + if (ext === ".png") contentType = "image/png"; + else if (ext === ".jpg" || ext === ".jpeg") contentType = "image/jpeg"; + else if (ext === ".gif") contentType = "image/gif"; + else if (ext === ".svg") contentType = "image/svg+xml"; + else if (ext === ".webp") contentType = "image/webp"; + + log(config, "debug", `Uploading ${localFilePath} to Heretto file ${fileId}`); + + const response = await client.put( + `${REST_API_PATH}/${fileId}/content`, + fileBuffer, + { + headers: { + "Content-Type": contentType, + "Content-Length": fileBuffer.length, + }, + maxBodyLength: Infinity, + maxContentLength: Infinity, + } + ); + + if (response.status === 200 || response.status === 201) { + log( + config, + "info", + `Successfully uploaded ${path.basename(localFilePath)} to Heretto` + ); + return { + status: "PASS", + description: `File uploaded successfully to Heretto`, + }; + } + + return { + status: "FAIL", + description: `Unexpected response status: ${response.status}`, + }; + } catch (error) { + const errorMessage = error.response?.data || error.message; + log( + config, + "warning", + `Failed to upload file to Heretto: ${errorMessage}` + ); + return { + status: "FAIL", + description: `Failed to upload: ${errorMessage}`, + }; + } +} + +/** + * Resolves a local file path to a Heretto file ID. + * First checks file mapping, then searches by filename if needed. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {string} localFilePath - Local path to the file + * @param {Object} sourceIntegration - Source integration metadata from step + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Heretto file ID or null if not found + */ +async function resolveFileId( + herettoConfig, + localFilePath, + sourceIntegration, + log, + config +) { + // If fileId is already known, use it + if (sourceIntegration?.fileId) { + return sourceIntegration.fileId; + } + + // Check file mapping + if (herettoConfig.fileMapping && herettoConfig.fileMapping[localFilePath]) { + const mapping = herettoConfig.fileMapping[localFilePath]; + if (mapping.fileId) { + return mapping.fileId; + } + } + + // Search by filename + const filename = path.basename(localFilePath); + const searchResult = await searchFileByName( + herettoConfig, + filename, + null, + log, + config + ); + + if (searchResult?.fileId) { + // Cache the result in file mapping + if (!herettoConfig.fileMapping) { + herettoConfig.fileMapping = {}; + } + herettoConfig.fileMapping[localFilePath] = { + fileId: searchResult.fileId, + filePath: searchResult.filePath, + }; + return searchResult.fileId; + } + + log( + config, + "warning", + `Could not resolve Heretto file ID for ${localFilePath}` + ); + return null; +} + module.exports = { createAuthHeader, createApiClient, + createRestApiClient, findScenario, triggerPublishingJob, pollJobStatus, downloadAndExtractOutput, loadHerettoContent, + buildFileMapping, + searchFileByName, + uploadFile, + resolveFileId, // Export constants for testing POLLING_INTERVAL_MS, POLLING_TIMEOUT_MS, diff --git a/src/utils.js b/src/utils.js index 4c32bb9..57acea8 100644 --- a/src/utils.js +++ b/src/utils.js @@ -26,6 +26,28 @@ exports.cleanTemp = cleanTemp; exports.calculatePercentageDifference = calculatePercentageDifference; exports.fetchFile = fetchFile; exports.isRelativeUrl = isRelativeUrl; +exports.findHerettoIntegration = findHerettoIntegration; + +/** + * Finds which Heretto integration a file belongs to based on its path. + * @param {Object} config - Doc Detective config with _herettoPathMapping + * @param {string} filePath - Path to check + * @returns {string|null} Heretto integration name or null if not from Heretto + */ +function findHerettoIntegration(config, filePath) { + if (!config._herettoPathMapping) return null; + + const normalizedFilePath = path.resolve(filePath); + + for (const [outputPath, integrationName] of Object.entries(config._herettoPathMapping)) { + const normalizedOutputPath = path.resolve(outputPath); + if (normalizedFilePath.startsWith(normalizedOutputPath)) { + return integrationName; + } + } + + return null; +} function isRelativeUrl(url) { try { @@ -230,6 +252,11 @@ async function qualifyFiles({ config }) { } const ignoredDitaMaps = []; + + // Track Heretto output paths for sourceIntegration metadata + if (!config._herettoPathMapping) { + config._herettoPathMapping = {}; + } for (let source of sequence) { log(config, "debug", `source: ${source}`); @@ -256,6 +283,8 @@ async function qualifyFiles({ config }) { const outputPath = await loadHerettoContent(herettoConfig, log, config); if (outputPath) { herettoConfig.outputPath = outputPath; + // Store mapping from output path to Heretto integration name + config._herettoPathMapping[outputPath] = herettoName; log(config, "debug", `Adding Heretto output path: ${outputPath}`); // Insert the output path into the sequence for processing const currentIndex = sequence.indexOf(source); @@ -728,10 +757,47 @@ async function parseContent({ config, content, filePath, fileType }) { ) { step[action].origin = config.origin; } + // Attach sourceIntegration metadata for screenshot steps from Heretto + if (action === "screenshot" && config._herettoPathMapping) { + const herettoIntegration = findHerettoIntegration(config, filePath); + if (herettoIntegration) { + // Convert simple screenshot value to object with sourceIntegration + const screenshotPath = step[action]; + step[action] = { + path: screenshotPath, + sourceIntegration: { + type: "heretto", + integrationName: herettoIntegration, + filePath: screenshotPath, + contentPath: filePath, + }, + }; + } + } } else { // Substitute variables $n with match[n] // TODO: Make key substitution recursive step = replaceNumericVariables(action, statement); + + // Attach sourceIntegration metadata for screenshot steps from Heretto + if (step.screenshot && config._herettoPathMapping) { + const herettoIntegration = findHerettoIntegration(config, filePath); + if (herettoIntegration) { + // Ensure screenshot is an object + if (typeof step.screenshot === "string") { + step.screenshot = { path: step.screenshot }; + } else if (typeof step.screenshot === "boolean") { + step.screenshot = {}; + } + // Attach sourceIntegration + step.screenshot.sourceIntegration = { + type: "heretto", + integrationName: herettoIntegration, + filePath: step.screenshot.path || "", + contentPath: filePath, + }; + } + } } // Normalize step field formats From dcb3022489eec4f13cc025a65aac9ddb0c9e41ca Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 19 Dec 2025 18:23:20 -0800 Subject: [PATCH 2/9] Add -based detection --- src/config.js | 53 +++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/src/config.js b/src/config.js index 8bacb3a..b363a2e 100644 --- a/src/config.js +++ b/src/config.js @@ -88,6 +88,7 @@ let defaultFileTypes = { step: [ "<\\?doc-detective\\s+step\\s+([\\s\\S]*?)\\s*\\?>", "", + '([\\s\\S]*?)<\\/data>', ], }, markup: [ @@ -124,7 +125,7 @@ let defaultFileTypes = { { name: "runShellCmdWithCodeblock", regex: [ - "(?:[Rr]un|[Ee]xecute)\\s+(?:the\\s+)?(?:following\\s+)?(?:command)[^<]*<\\/cmd>\\s*\\s*]*outputclass=\"(?:shell|bash)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + '(?:[Rr]un|[Ee]xecute)\\s+(?:the\\s+)?(?:following\\s+)?(?:command)[^<]*<\\/cmd>\\s*\\s*]*outputclass="(?:shell|bash)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -133,27 +134,21 @@ let defaultFileTypes = { }, }, ], - }, + }, // Inline Elements - for finding UI elements and text { name: "findUiControl", - regex: [ - "([^<]+)<\\/uicontrol>", - ], + regex: ["([^<]+)<\\/uicontrol>"], actions: ["find"], }, { name: "verifyWindowTitle", - regex: [ - "([^<]+)<\\/wintitle>", - ], + regex: ["([^<]+)<\\/wintitle>"], actions: ["find"], }, { name: "EnterKey", - regex: [ - "(?:[Pp]ress)\\s+Enter<\\/shortcut>", - ], + regex: ["(?:[Pp]ress)\\s+Enter<\\/shortcut>"], actions: [ { type: { @@ -164,9 +159,7 @@ let defaultFileTypes = { }, { name: "executeCmdName", - regex: [ - "(?:[Ee]xecute|[Rr]un)\\s+([^<]+)<\\/cmdname>", - ], + regex: ["(?:[Ee]xecute|[Rr]un)\\s+([^<]+)<\\/cmdname>"], actions: [ { runShell: { @@ -175,7 +168,7 @@ let defaultFileTypes = { }, ], }, - + // Links and References - for link validation { name: "checkExternalXref", @@ -187,24 +180,20 @@ let defaultFileTypes = { }, { name: "checkHyperlink", - regex: [ - ']*>', - ], + regex: [']*>'], actions: ["checkLink"], }, { name: "checkLinkElement", - regex: [ - ']*>', - ], + regex: [']*>'], actions: ["checkLink"], }, - + // Code Execution { name: "runShellCodeblock", regex: [ - "]*outputclass=\"(?:shell|bash)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + ']*outputclass="(?:shell|bash)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -217,7 +206,7 @@ let defaultFileTypes = { { name: "runCode", regex: [ - "]*outputclass=\"(python|py|javascript|js)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + ']*outputclass="(python|py|javascript|js)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -231,7 +220,7 @@ let defaultFileTypes = { }, ], }, - + // Legacy patterns for compatibility with existing tests { name: "clickOnscreenText", @@ -255,8 +244,10 @@ let defaultFileTypes = { { name: "screenshotImage", regex: [ + ']*outputclass="[^"]*screenshot[^"]*href="([^"]+)"[^>]*"[^>]*\\/>', ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[^>]*\\/>', - ], + ']*outputclass="[^"]*screenshot[^"]*"[^>]*href="([^"]+)"[\\s\\S]*?<\\/image>', + ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[\\s\\S]*?<\\/image>', ], actions: ["screenshot"], }, { @@ -267,7 +258,7 @@ let defaultFileTypes = { { name: "httpRequestFormat", regex: [ - "]*outputclass=\"http\"[^>]*>\\s*([A-Z]+)\\s+([^\\s]+)(?:\\s+HTTP\\/[\\d.]+)?\\s*(?:\\r?\\n| )((?:[^\\s<]+:\\s+[^\\r\\n<]+(?:\\r?\\n| ))*)(?:\\s*(?:\\r?\\n| )([\\s\\S]*?))?\\s*<\\/codeblock>", + ']*outputclass="http"[^>]*>\\s*([A-Z]+)\\s+([^\\s]+)(?:\\s+HTTP\\/[\\d.]+)?\\s*(?:\\r?\\n| )((?:[^\\s<]+:\\s+[^\\r\\n<]+(?:\\r?\\n| ))*)(?:\\s*(?:\\r?\\n| )([\\s\\S]*?))?\\s*<\\/codeblock>', ], actions: [ { @@ -285,7 +276,7 @@ let defaultFileTypes = { { name: "runCode", regex: [ - "]*outputclass=\"(bash|python|py|javascript|js)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + ']*outputclass="(bash|python|py|javascript|js)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -462,7 +453,7 @@ defaultFileTypes = { /** * Resolves the concurrentRunners configuration value from various input formats * to a concrete integer for the core execution engine. - * + * * @param {Object} config - The configuration object * @returns {number} The resolved concurrent runners value */ @@ -665,10 +656,10 @@ async function setConfig({ config }) { // Detect current environment. config.environment = getEnvironment(); - + // Resolve concurrent runners configuration config.concurrentRunners = resolveConcurrentRunners(config); - + // TODO: Revise loadDescriptions() so it doesn't mutate the input but instead returns an updated object await loadDescriptions(config); From 97e10a3d48987f564ee3db13e38569951233bcb0 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 19 Dec 2025 18:24:21 -0800 Subject: [PATCH 3/9] Set spec ID based on file name --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 57acea8..099feb0 100644 --- a/src/utils.js +++ b/src/utils.js @@ -978,7 +978,7 @@ async function parseTests({ config, files }) { specs.push(content); } else { // Process non-object - let id = `${crypto.randomUUID()}`; + let id = path.basename(file); let spec = { specId: id, contentPath: file, tests: [] }; const fileType = config.fileTypes.find((fileType) => fileType.extensions.includes(extension) From d7cfb3d492a7693b41fc485bcb8513941ac4ef4e Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Mon, 22 Dec 2025 12:41:30 -0800 Subject: [PATCH 4/9] feat: Add Heretto integration tests and enhance job asset handling - Create integration tests for Heretto API interactions. - Implement job asset retrieval with pagination support. - Validate presence of .ditamap files in job assets. - Update package.json scripts for integration testing. - Modify Heretto functions to improve error handling and logging. --- .github/workflows/integration-tests.yml | 119 ++++++++++++ package.json | 6 +- src/heretto.integration.test.js | 232 +++++++++++++++++++++++ src/heretto.js | 116 +++++++++++- src/heretto.test.js | 242 ++++++++++++++++++++++-- 5 files changed, 691 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/integration-tests.yml create mode 100644 src/heretto.integration.test.js diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000..a372c93 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,119 @@ +name: Integration Tests + +on: + push: + branches: + - main + - heretto + paths: + - 'src/heretto*.js' + - '.github/workflows/integration-tests.yml' + pull_request: + branches: + - main + paths: + - 'src/heretto*.js' + - '.github/workflows/integration-tests.yml' + workflow_dispatch: + # Allow manual triggering for testing + schedule: + # Run daily at 6:00 AM UTC to catch any API changes + - cron: '0 6 * * *' + +jobs: + heretto-integration-tests: + runs-on: ubuntu-latest + timeout-minutes: 15 + # Only run if secrets are available (not available on fork PRs) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run integration tests + env: + CI: 'true' + HERETTO_ORGANIZATION_ID: ${{ secrets.HERETTO_ORGANIZATION_ID }} + HERETTO_USERNAME: ${{ secrets.HERETTO_USERNAME }} + HERETTO_API_TOKEN: ${{ secrets.HERETTO_API_TOKEN }} + HERETTO_SCENARIO_NAME: ${{ secrets.HERETTO_SCENARIO_NAME || 'Doc Detective' }} + run: npm run test:integration + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-results + path: | + test-results/ + *.log + retention-days: 7 + + notify-on-failure: + runs-on: ubuntu-latest + needs: heretto-integration-tests + if: failure() && github.event_name == 'schedule' + steps: + - name: Create issue on failure + uses: actions/github-script@v7 + with: + script: | + const title = '🚨 Heretto Integration Tests Failed'; + const body = ` + ## Integration Test Failure + + The scheduled Heretto integration tests have failed. + + **Workflow Run:** [View Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + **Triggered:** ${{ github.event_name }} + **Branch:** ${{ github.ref_name }} + + Please investigate and fix the issue. + + ### Possible Causes + - Heretto API changes + - Expired or invalid API credentials + - Network connectivity issues + - Changes in test scenario configuration + + /cc @${{ github.repository_owner }} + `; + + // Check if an open issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'integration-test-failure' + }); + + const existingIssue = issues.data.find(issue => issue.title === title); + + if (!existingIssue) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['bug', 'integration-test-failure', 'automated'] + }); + } else { + // Add a comment to the existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existingIssue.number, + body: `Another failure detected on ${new Date().toISOString()}\n\n[Workflow Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})` + }); + } diff --git a/package.json b/package.json index f74d3f3..2aa74f4 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,9 @@ "description": "Detect and resolve docs into Doc Detective tests.", "main": "src/index.js", "scripts": { - "test": "mocha src/*.test.js", + "test": "mocha src/*.test.js --ignore src/*.integration.test.js", + "test:integration": "mocha src/*.integration.test.js --timeout 600000", + "test:all": "mocha src/*.test.js --timeout 600000", "dev": "node dev" }, "repository": { @@ -28,7 +30,7 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "3.6.0-dev.1", + "doc-detective-common": "file:../common", "dotenv": "^17.2.3", "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", diff --git a/src/heretto.integration.test.js b/src/heretto.integration.test.js new file mode 100644 index 0000000..a26e987 --- /dev/null +++ b/src/heretto.integration.test.js @@ -0,0 +1,232 @@ +/** + * Heretto Integration Tests + * + * These tests run against the real Heretto API and are designed to only + * execute in CI environments (GitHub Actions) where credentials are available. + * + * Required environment variables: + * - HERETTO_ORGANIZATION_ID: The Heretto organization ID + * - HERETTO_USERNAME: The Heretto username (email) + * - HERETTO_API_TOKEN: The Heretto API token + * + * These tests are skipped when: + * - Running locally without CI=true environment variable + * - Required environment variables are not set + */ + +const heretto = require("./heretto"); + +before(async function () { + const { expect } = await import("chai"); + global.expect = expect; +}); + +/** + * Check if we're running in CI and have required credentials + */ +const isCI = process.env.CI === "true"; +const hasCredentials = + process.env.HERETTO_ORGANIZATION_ID && + process.env.HERETTO_USERNAME && + process.env.HERETTO_API_TOKEN; + +const shouldRunIntegrationTests = isCI && hasCredentials; + +// Helper to skip tests when not in CI or missing credentials +const describeIntegration = shouldRunIntegrationTests ? describe : describe.skip; + +// Log why tests are being skipped +if (!shouldRunIntegrationTests) { + console.log("\n⏭️ Heretto integration tests skipped:"); + if (!isCI) { + console.log(" - Not running in CI environment (CI !== 'true')"); + } + if (!hasCredentials) { + console.log(" - Missing required environment variables:"); + if (!process.env.HERETTO_ORGANIZATION_ID) + console.log(" - HERETTO_ORGANIZATION_ID"); + if (!process.env.HERETTO_USERNAME) console.log(" - HERETTO_USERNAME"); + if (!process.env.HERETTO_API_TOKEN) console.log(" - HERETTO_API_TOKEN"); + } + console.log(""); +} + +describeIntegration("Heretto Integration Tests (CI Only)", function () { + // These tests interact with real APIs, so allow longer timeouts + this.timeout(120000); // 2 minutes per test + + let client; + let herettoConfig; + const mockLog = (...args) => { + if (process.env.DEBUG) { + console.log(...args); + } + }; + const mockConfig = { logLevel: process.env.DEBUG ? "debug" : "info" }; + + before(function () { + herettoConfig = { + name: "integration-test", + organizationId: process.env.HERETTO_ORGANIZATION_ID, + username: process.env.HERETTO_USERNAME, + apiToken: process.env.HERETTO_API_TOKEN, + scenarioName: process.env.HERETTO_SCENARIO_NAME || "Doc Detective", + }; + + client = heretto.createApiClient(herettoConfig); + }); + + describe("API Client Creation", function () { + it("should create a valid API client", function () { + expect(client).to.not.be.null; + expect(client).to.have.property("get"); + expect(client).to.have.property("post"); + }); + + it("should configure correct base URL", function () { + const expectedBaseUrl = `https://${herettoConfig.organizationId}.heretto.com/ezdnxtgen/api/v2`; + expect(client.defaults.baseURL).to.equal(expectedBaseUrl); + }); + }); + + describe("findScenario", function () { + it("should find an existing scenario with correct configuration", async function () { + const result = await heretto.findScenario( + client, + mockLog, + mockConfig, + herettoConfig.scenarioName + ); + + // The scenario should exist and have required properties + expect(result).to.not.be.null; + expect(result).to.have.property("scenarioId"); + expect(result).to.have.property("fileId"); + expect(result.scenarioId).to.be.a("string"); + expect(result.fileId).to.be.a("string"); + }); + + it("should return null for non-existent scenario", async function () { + const result = await heretto.findScenario( + client, + mockLog, + mockConfig, + "NonExistent Scenario That Should Not Exist 12345" + ); + + expect(result).to.be.null; + }); + }); + + describe("Full Publishing Workflow", function () { + let scenarioInfo; + let jobId; + + before(async function () { + // Find the scenario first + scenarioInfo = await heretto.findScenario( + client, + mockLog, + mockConfig, + herettoConfig.scenarioName + ); + + if (!scenarioInfo) { + this.skip(); + } + }); + + it("should trigger a publishing job", async function () { + const job = await heretto.triggerPublishingJob( + client, + scenarioInfo.fileId, + scenarioInfo.scenarioId + ); + + expect(job).to.not.be.null; + expect(job).to.have.property("id"); + jobId = job.id; + }); + + it("should poll job status until completion", async function () { + // This test may take a while as it waits for the job to complete + this.timeout(360000); // 6 minutes + + const completedJob = await heretto.pollJobStatus( + client, + scenarioInfo.fileId, + jobId, + mockLog, + mockConfig + ); + + expect(completedJob).to.not.be.null; + expect(completedJob).to.have.property("status"); + expect(completedJob.status).to.have.property("status"); + + // Job should be in a completed state + const completedStates = ["COMPLETED", "FAILED", "DONE"]; + expect(completedStates).to.include(completedJob.status.status); + }); + + it("should fetch job asset details", async function () { + const assets = await heretto.getJobAssetDetails( + client, + scenarioInfo.fileId, + jobId + ); + + expect(assets).to.be.an("array"); + expect(assets.length).to.be.greaterThan(0); + + // Should contain at least some DITA files + const hasDitaFiles = assets.some( + (path) => path.endsWith(".dita") || path.endsWith(".ditamap") + ); + expect(hasDitaFiles).to.be.true; + }); + + it("should validate ditamap exists in assets", async function () { + const assets = await heretto.getJobAssetDetails( + client, + scenarioInfo.fileId, + jobId + ); + + const hasValidDitamap = heretto.validateDitamapInAssets(assets); + expect(hasValidDitamap).to.be.true; + }); + + it("should download and extract output", async function () { + const outputPath = await heretto.downloadAndExtractOutput( + client, + scenarioInfo.fileId, + jobId, + herettoConfig.name, + mockLog, + mockConfig + ); + + expect(outputPath).to.not.be.null; + expect(outputPath).to.be.a("string"); + expect(outputPath).to.include("heretto_"); + }); + }); + + describe("loadHerettoContent (End-to-End)", function () { + it("should load content from Heretto successfully", async function () { + // This is the full end-to-end test + this.timeout(600000); // 10 minutes for full workflow + + const outputPath = await heretto.loadHerettoContent( + herettoConfig, + mockLog, + mockConfig + ); + + expect(outputPath).to.not.be.null; + expect(outputPath).to.be.a("string"); + expect(outputPath).to.include("heretto_"); + }); + }); +}); diff --git a/src/heretto.js b/src/heretto.js index 968b8bb..b64513f 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -173,7 +173,10 @@ async function findScenario(client, log, config, scenarioName) { "debug", `Found existing "${scenarioName}" scenario: ${foundScenario.id}` ); - return { scenarioId: foundScenario.id, fileId: fileUuidPickerParam.value }; + return { + scenarioId: foundScenario.id, + fileId: fileUuidPickerParam.value, + }; } catch (error) { log( config, @@ -213,8 +216,64 @@ async function getJobStatus(client, fileId, jobId) { return response.data; } +/** + * Gets all asset file paths from a completed publishing job. + * Handles pagination to retrieve all assets. + * @param {Object} client - Configured axios instance + * @param {string} fileId - UUID of the DITA map + * @param {string} jobId - ID of the publishing job + * @returns {Promise>} Array of asset file paths + */ +async function getJobAssetDetails(client, fileId, jobId) { + const allAssets = []; + let page = 0; + const pageSize = 100; + let hasMorePages = true; + + while (hasMorePages) { + const response = await client.get( + `/files/${fileId}/publishes/${jobId}/assets`, + { + params: { + page, + size: pageSize, + }, + } + ); + + const data = response.data; + const content = data.content || []; + + for (const asset of content) { + if (asset.filePath) { + allAssets.push(asset.filePath); + } + } + + // Check if there are more pages + const totalPages = data.totalPages || 1; + page++; + hasMorePages = page < totalPages; + } + + return allAssets; +} + +/** + * Validates that a .ditamap file exists in the job assets. + * Checks for any .ditamap file in the ot-output/dita/ directory. + * @param {Array} assets - Array of asset file paths + * @returns {boolean} True if a .ditamap is found in ot-output/dita/ + */ +function validateDitamapInAssets(assets) { + return assets.some((assetPath) => + assetPath.startsWith("ot-output/dita/") && assetPath.endsWith(".ditamap") + ); +} + /** * Polls a publishing job until completion or timeout. + * After job completes, validates that a .ditamap file exists in the output. * @param {Object} client - Configured axios instance * @param {string} fileId - UUID of the DITA map * @param {string} jobId - ID of the publishing job @@ -230,17 +289,46 @@ async function pollJobStatus(client, fileId, jobId, log, config) { const job = await getJobStatus(client, fileId, jobId); log(config, "debug", `Job ${jobId} status: ${job?.status?.status}`); - if (job?.status?.result === "SUCCESS") { - return job; - } - - if (job?.status?.result === "FAIL") { + // Check if job has reached a terminal state (result is set) + if (job?.status?.result) { log( config, - "warning", - `Publishing job ${jobId} failed.` + "debug", + `Job ${jobId} completed with result: ${job.status.result}` ); - return null; + + // Validate that a .ditamap file exists in the output + try { + const assets = await getJobAssetDetails(client, fileId, jobId); + log( + config, + "debug", + `Job ${jobId} has ${assets.length} assets` + ); + + if (validateDitamapInAssets(assets)) { + log( + config, + "debug", + `Found .ditamap file in ot-output/dita/` + ); + return job; + } + + log( + config, + "warning", + `Publishing job ${jobId} completed but no .ditamap file found in ot-output/dita/` + ); + return null; + } catch (assetError) { + log( + config, + "warning", + `Failed to validate job assets: ${assetError.message}` + ); + return null; + } } // Wait before next poll @@ -375,7 +463,12 @@ async function loadHerettoContent(herettoConfig, log, config) { // Find the Doc Detective publishing scenario const scenarioName = herettoConfig.scenarioName || DEFAULT_SCENARIO_NAME; - const scenario = await findScenario(client, log, config, scenarioName); + const scenario = await findScenario( + client, + log, + config, + scenarioName + ); if (!scenario) { log( config, @@ -789,6 +882,9 @@ module.exports = { createRestApiClient, findScenario, triggerPublishingJob, + getJobStatus, + getJobAssetDetails, + validateDitamapInAssets, pollJobStatus, downloadAndExtractOutput, loadHerettoContent, diff --git a/src/heretto.test.js b/src/heretto.test.js index c89ea82..c47a5a8 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -100,9 +100,11 @@ describe("Heretto Integration", function () { const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); - expect(result).to.deep.equal({ scenarioId: "scenario-123", fileId: "file-uuid-456" }); + expect(result).to.deep.equal({ + scenarioId: "scenario-123", + fileId: "file-uuid-456", + }); expect(mockClient.get.calledTwice).to.be.true; - expect(mockClient.post.called).to.be.false; }); it("should return null if scenario is not found", async function () { @@ -114,7 +116,6 @@ describe("Heretto Integration", function () { expect(result).to.be.null; expect(mockClient.get.calledOnce).to.be.true; - expect(mockClient.post.called).to.be.false; }); it("should return null if scenario fetch fails", async function () { @@ -228,6 +229,157 @@ describe("Heretto Integration", function () { }); }); + describe("getJobAssetDetails", function () { + it("should return all asset file paths from single page", async function () { + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + { filePath: "ot-output/dita/topic1.dita" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 1, + number: 0, + size: 100, + }; + + mockClient.get.resolves({ data: assetsResponse }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([ + "ot-output/dita/my-guide.ditamap", + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + ]); + expect(mockClient.get.calledOnce).to.be.true; + expect(mockClient.get.firstCall.args[0]).to.equal("/files/file-uuid/publishes/job-123/assets"); + }); + + it("should handle pagination and aggregate all assets", async function () { + const page1Response = { + content: [ + { filePath: "ot-output/dita/topic1.dita" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 2, + number: 0, + size: 100, + }; + + const page2Response = { + content: [ + { filePath: "ot-output/dita/topic3.dita" }, + { filePath: "ot-output/dita/my-guide.ditamap" }, + ], + totalPages: 2, + number: 1, + size: 100, + }; + + mockClient.get + .onFirstCall().resolves({ data: page1Response }) + .onSecondCall().resolves({ data: page2Response }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([ + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + "ot-output/dita/topic3.dita", + "ot-output/dita/my-guide.ditamap", + ]); + expect(mockClient.get.calledTwice).to.be.true; + }); + + it("should return empty array when no assets", async function () { + const assetsResponse = { + content: [], + totalPages: 1, + number: 0, + size: 100, + }; + + mockClient.get.resolves({ data: assetsResponse }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([]); + }); + + it("should skip assets without filePath", async function () { + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/topic1.dita" }, + { otherField: "no-path" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 1, + }; + + mockClient.get.resolves({ data: assetsResponse }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([ + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + ]); + }); + }); + + describe("validateDitamapInAssets", function () { + it("should return true when ditamap is in ot-output/dita/", function () { + const assets = [ + "ot-output/dita/topic1.dita", + "ot-output/dita/my-guide.ditamap", + "ot-output/dita/topic2.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.true; + }); + + it("should return false when no ditamap is present", function () { + const assets = [ + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.false; + }); + + it("should return false when ditamap is in wrong directory", function () { + const assets = [ + "ot-output/other/my-guide.ditamap", + "ot-output/dita/topic1.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.false; + }); + + it("should return true when any ditamap is in correct directory", function () { + const assets = [ + "ot-output/dita/different-guide.ditamap", + "ot-output/dita/topic1.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.true; + }); + + it("should return false when assets array is empty", function () { + const result = heretto.validateDitamapInAssets([]); + + expect(result).to.be.false; + }); + }); + describe("pollJobStatus", function () { const mockLog = sinon.stub(); const mockConfig = { logLevel: "info" }; @@ -236,40 +388,91 @@ describe("Heretto Integration", function () { mockLog.reset(); }); - it("should return completed job when status.result is SUCCESS", async function () { + it("should return completed job when status.result is SUCCESS and ditamap is present", async function () { const completedJob = { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" }, }; - mockClient.get.resolves({ data: completedJob }); + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + { filePath: "ot-output/dita/topic1.dita" }, + ], + totalPages: 1, + }; + + mockClient.get + .onFirstCall().resolves({ data: completedJob }) + .onSecondCall().resolves({ data: assetsResponse }); const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); expect(result).to.deep.equal(completedJob); }); - it("should return null when status.result is FAIL", async function () { + it("should return completed job when status.result is FAIL but ditamap is present", async function () { const failedJob = { id: "job-123", status: { status: "FAILED", result: "FAIL" }, }; - mockClient.get.resolves({ data: failedJob }); + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + { filePath: "ot-output/dita/topic1.dita" }, + ], + totalPages: 1, + }; + + mockClient.get + .onFirstCall().resolves({ data: failedJob }) + .onSecondCall().resolves({ data: assetsResponse }); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.deep.equal(failedJob); + }); + + it("should return null when job completes but ditamap is missing", async function () { + const completedJob = { + id: "job-123", + status: { status: "COMPLETED", result: "SUCCESS" }, + }; + + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/topic1.dita" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 1, + }; + + mockClient.get + .onFirstCall().resolves({ data: completedJob }) + .onSecondCall().resolves({ data: assetsResponse }); const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); expect(result).to.be.null; }); - it("should poll until completion", async function () { + it("should poll until completion then validate assets", async function () { // Use fake timers to avoid waiting for real POLLING_INTERVAL_MS delays const clock = sinon.useFakeTimers(); + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + ], + totalPages: 1, + }; + mockClient.get - .onFirstCall().resolves({ data: { id: "job-123", status: { status: "PENDING", result: null } } }) - .onSecondCall().resolves({ data: { id: "job-123", status: { status: "PROCESSING", result: null } } }) - .onThirdCall().resolves({ data: { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" } } }); + .onCall(0).resolves({ data: { id: "job-123", status: { status: "PENDING", result: null } } }) + .onCall(1).resolves({ data: { id: "job-123", status: { status: "PROCESSING", result: null } } }) + .onCall(2).resolves({ data: { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" } } }) + .onCall(3).resolves({ data: assetsResponse }); const pollPromise = heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); @@ -281,7 +484,7 @@ describe("Heretto Integration", function () { const result = await pollPromise; expect(result.status.result).to.equal("SUCCESS"); - expect(mockClient.get.callCount).to.equal(3); + expect(mockClient.get.callCount).to.equal(4); // 3 status polls + 1 assets call clock.restore(); }); @@ -313,6 +516,21 @@ describe("Heretto Integration", function () { expect(result).to.be.null; }); + + it("should return null when asset validation fails", async function () { + const completedJob = { + id: "job-123", + status: { status: "COMPLETED", result: "SUCCESS" }, + }; + + mockClient.get + .onFirstCall().resolves({ data: completedJob }) + .onSecondCall().rejects(new Error("Failed to fetch assets")); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.be.null; + }); }); describe("loadHerettoContent", function () { From d7ebd35f9d8e62d098414754b1a5c9cac674ce28 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Mon, 22 Dec 2025 15:40:40 -0800 Subject: [PATCH 5/9] Implement screenshot handling and upload functionality - Update `saveScreenshot.js` to mark new screenshots as changed for upload. - Add a new test file `screenshot.test.js` to verify sourceIntegration preservation and changed status for screenshots. - Create a new test file `upload.test.js` to validate the upload module, including collecting changed files and integration configuration. - Modify `package-lock.json` to link to local `doc-detective-common` instead of a versioned package. - Enhance `heretto.js` to retrieve resource dependencies from the Heretto REST API, including detailed logging and XML parsing. --- package-lock.json | 76 ++++++++--------------- src/heretto.js | 152 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 175 insertions(+), 53 deletions(-) diff --git a/package-lock.json b/package-lock.json index abff311..173a859 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "3.6.0-dev.1", + "doc-detective-common": "file:../common", "dotenv": "^17.2.3", "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", @@ -30,6 +30,25 @@ "yaml": "^2.8.2" } }, + "../common": { + "name": "doc-detective-common", + "version": "3.6.0-dev.1", + "license": "AGPL-3.0-only", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^15.1.3", + "ajv": "^8.17.1", + "ajv-errors": "^3.0.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0", + "axios": "^1.13.2", + "yaml": "^2.8.2" + }, + "devDependencies": { + "chai": "^6.2.1", + "mocha": "^11.7.5", + "sinon": "^21.0.0" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "15.1.3", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-15.1.3.tgz", @@ -244,44 +263,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ajv-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-3.0.0.tgz", - "integrity": "sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==", - "license": "MIT", - "peerDependencies": { - "ajv": "^8.0.1" - } - }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ajv-keywords": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", - "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3" - }, - "peerDependencies": { - "ajv": "^8.8.2" - } - }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -713,19 +694,8 @@ } }, "node_modules/doc-detective-common": { - "version": "3.6.0-dev.1", - "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.6.0-dev.1.tgz", - "integrity": "sha512-e+3FNyqjhPUZRq+4A1t7G+au07RZockzCHdQ6LDaQQySPtAiNSO42v48ylbHIu4ZOn06SO933rVJe/b+e1GVdw==", - "license": "AGPL-3.0-only", - "dependencies": { - "@apidevtools/json-schema-ref-parser": "^15.1.3", - "ajv": "^8.17.1", - "ajv-errors": "^3.0.0", - "ajv-formats": "^3.0.1", - "ajv-keywords": "^5.1.0", - "axios": "^1.13.2", - "yaml": "^2.8.2" - } + "resolved": "../common", + "link": true }, "node_modules/dotenv": { "version": "17.2.3", @@ -1401,6 +1371,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -2617,6 +2588,7 @@ "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "dev": true, "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/src/heretto.js b/src/heretto.js index b64513f..a5d9052 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -70,6 +70,7 @@ function createRestApiClient(herettoConfig) { timeout: API_REQUEST_TIMEOUT_MS, headers: { Authorization: `Basic ${authHeader}`, + Accept: "application/xml, text/xml, */*", }, }); } @@ -443,6 +444,140 @@ async function downloadAndExtractOutput( } } +/** + * Retrieves resource dependencies (all files) for a ditamap from Heretto REST API. + * This provides the complete file structure with UUIDs and paths. + * @param {Object} restClient - Configured axios instance for REST API + * @param {string} ditamapId - UUID of the ditamap file + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Object mapping relative paths to UUIDs and parent folder info + */ +async function getResourceDependencies(restClient, ditamapId, log, config) { + const pathToUuidMap = {}; + + const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + }); + + // First, try to get the ditamap's own info (this is more reliable than the dependencies endpoint) + try { + log(config, "debug", `Fetching ditamap info for: ${ditamapId}`); + const ditamapInfo = await restClient.get(`${REST_API_PATH}/${ditamapId}`); + const ditamapParsed = xmlParser.parse(ditamapInfo.data); + + const ditamapUri = ditamapParsed.resource?.["xmldb-uri"] || ditamapParsed["@_uri"]; + const ditamapName = ditamapParsed.resource?.name || ditamapParsed["@_name"]; + const ditamapParentFolder = ditamapParsed.resource?.["folder-uuid"] || + ditamapParsed.resource?.["@_folder-uuid"] || + ditamapParsed["@_folder-uuid"]; + + log(config, "debug", `Ditamap info: uri=${ditamapUri}, name=${ditamapName}, parentFolder=${ditamapParentFolder}`); + + if (ditamapUri) { + let relativePath = ditamapUri; + const orgPathMatch = relativePath?.match(/\/db\/organizations\/[^/]+\/(.+)/); + if (orgPathMatch) { + relativePath = orgPathMatch[1]; + } + + pathToUuidMap[relativePath] = { + uuid: ditamapId, + fullPath: ditamapUri, + name: ditamapName, + parentFolderId: ditamapParentFolder, + isDitamap: true, + }; + + // Store the ditamap info as reference points for creating new files + pathToUuidMap._ditamapPath = relativePath; + pathToUuidMap._ditamapId = ditamapId; + pathToUuidMap._ditamapParentFolderId = ditamapParentFolder; + + log(config, "debug", `Ditamap path: ${relativePath}, parent folder: ${ditamapParentFolder}`); + } + } catch (ditamapError) { + log(config, "warning", `Could not get ditamap info: ${ditamapError.message}`); + } + + // Then try to get the full dependencies list (this endpoint may not be available) + try { + log(config, "debug", `Fetching resource dependencies for ditamap: ${ditamapId}`); + + const response = await restClient.get(`${REST_API_PATH}/${ditamapId}/dependencies`); + const xmlData = response.data; + + const parsed = xmlParser.parse(xmlData); + + // Extract dependencies from the response + // Response format: ...... + const extractDependencies = (obj, parentPath = "") => { + if (!obj) return; + + // Handle single dependency or array of dependencies + let dependencies = obj.dependencies?.dependency || obj.dependency; + if (!dependencies) { + // Try to extract from root-level response + if (obj["@_id"] && obj["@_uri"]) { + dependencies = [obj]; + } else if (Array.isArray(obj)) { + dependencies = obj; + } + } + + if (!dependencies) return; + if (!Array.isArray(dependencies)) { + dependencies = [dependencies]; + } + + for (const dep of dependencies) { + const uuid = dep["@_id"] || dep["@_uuid"] || dep.id || dep.uuid; + const uri = dep["@_uri"] || dep["@_path"] || dep.uri || dep.path || dep["xmldb-uri"]; + const name = dep["@_name"] || dep.name; + const parentFolderId = dep["@_folder-uuid"] || dep["@_parent"] || dep["folder-uuid"]; + + if (uuid && (uri || name)) { + // Extract the relative path from the full URI + // URI format: /db/organizations/{org}/{path} + let relativePath = uri || name; + const orgPathMatch = relativePath?.match(/\/db\/organizations\/[^/]+\/(.+)/); + if (orgPathMatch) { + relativePath = orgPathMatch[1]; + } + + pathToUuidMap[relativePath] = { + uuid, + fullPath: uri, + name: name || path.basename(relativePath || ""), + parentFolderId, + }; + + log(config, "debug", `Mapped: ${relativePath} -> ${uuid}`); + } + + // Recursively process nested dependencies + if (dep.dependencies || dep.dependency) { + extractDependencies(dep); + } + } + }; + + extractDependencies(parsed); + + log(config, "info", `Retrieved ${Object.keys(pathToUuidMap).length} resource dependencies from Heretto`); + + } catch (error) { + // Log more details about the error for debugging + const statusCode = error.response?.status; + const responseData = error.response?.data; + log(config, "debug", `Dependencies endpoint not available (${statusCode}), will use ditamap info as fallback`); + // Continue with ditamap info only - the fallback will create files in the ditamap's parent folder + } + + return pathToUuidMap; +} + /** * Main function to load content from a Heretto CMS instance. * Triggers a publishing job, waits for completion, and downloads the output. @@ -460,6 +595,7 @@ async function loadHerettoContent(herettoConfig, log, config) { try { const client = createApiClient(herettoConfig); + const restClient = createRestApiClient(herettoConfig); // Find the Doc Detective publishing scenario const scenarioName = herettoConfig.scenarioName || DEFAULT_SCENARIO_NAME; @@ -478,6 +614,19 @@ async function loadHerettoContent(herettoConfig, log, config) { return null; } + // Fetch resource dependencies to build path-to-UUID mapping + // This gives us the complete file structure with UUIDs before we even run the job + if (herettoConfig.uploadOnChange) { + log(config, "debug", `Fetching resource dependencies for ditamap ${scenario.fileId}...`); + const resourceDependencies = await getResourceDependencies( + restClient, + scenario.fileId, + log, + config + ); + herettoConfig.resourceDependencies = resourceDependencies; + } + // Trigger publishing job log( config, @@ -519,7 +668,7 @@ async function loadHerettoContent(herettoConfig, log, config) { config ); - // Build file mapping from extracted content + // Build file mapping from extracted content (legacy approach, still useful as fallback) if (outputPath && herettoConfig.uploadOnChange) { const fileMapping = await buildFileMapping( outputPath, @@ -892,6 +1041,7 @@ module.exports = { searchFileByName, uploadFile, resolveFileId, + getResourceDependencies, // Export constants for testing POLLING_INTERVAL_MS, POLLING_TIMEOUT_MS, From 42d5da6915fcfcab1236b58709aa95b7a3cee6d8 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 30 Dec 2025 11:17:30 -0800 Subject: [PATCH 6/9] feat: Add generateSpecId function to create unique spec IDs from file paths - Implement generateSpecId to derive safe spec IDs from absolute or relative file paths. - Update parseTests to use generateSpecId for generating spec IDs, reducing collisions from files with the same basename. --- src/utils.js | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 099feb0..8517ad3 100644 --- a/src/utils.js +++ b/src/utils.js @@ -60,6 +60,33 @@ function isRelativeUrl(url) { } } +/** + * Generates a unique specId from a file path that is safe for storage/URLs. + * Uses relative path from cwd when possible to provide uniqueness while + * avoiding collisions from files with the same basename in different directories. + * @param {string} filePath - Absolute or relative file path + * @returns {string} A safe specId derived from the file path + */ +function generateSpecId(filePath) { + const absolutePath = path.resolve(filePath); + const cwd = process.cwd(); + + let relativePath; + if (absolutePath.startsWith(cwd)) { + relativePath = path.relative(cwd, absolutePath); + } else { + relativePath = absolutePath; + } + + const normalizedPath = relativePath + .split(path.sep) + .join("/") + .replace(/^\.\//, "") + .replace(/[^a-zA-Z0-9._\-\/]/g, "_"); + + return normalizedPath; +} + // Parse XML-style attributes to an object // Example: 'wait=500' becomes { wait: 500 } // Example: 'testId="myTestId" detectSteps=false' becomes { testId: "myTestId", detectSteps: false } @@ -978,7 +1005,9 @@ async function parseTests({ config, files }) { specs.push(content); } else { // Process non-object - let id = path.basename(file); + // Generate a specId that includes more of the file path to avoid collisions + // when different files share the same basename + let id = generateSpecId(file); let spec = { specId: id, contentPath: file, tests: [] }; const fileType = config.fileTypes.find((fileType) => fileType.extensions.includes(extension) From 67cd8a74c22537b13a70b903a3cdf0a0ac04cc3e Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 30 Dec 2025 11:17:35 -0800 Subject: [PATCH 7/9] fix: Correct API endpoint in searchFileByName function --- src/heretto.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heretto.js b/src/heretto.js index a5d9052..cedf1a2 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -861,7 +861,7 @@ async function searchFileByName( } const response = await client.post( - "/ezdnxtgen/api/search".replace("/ezdnxtgen/api/v2", ""), + "/ezdnxtgen/api/search", searchBody, { baseURL: `https://${herettoConfig.organizationId}.heretto.com`, From b7309fa44496161ca47b488847c682b296cb97dc Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Sat, 3 Jan 2026 13:49:51 -0800 Subject: [PATCH 8/9] chore: Update dependencies in package.json and package-lock.json - Change doc-detective-common from local file reference to version "^3.6.0-dev.2" - Update posthog-node to version "^5.18.1" - Upgrade chai to version "^6.2.2" - Upgrade sinon to version "^21.0.1" - Update ajv to version "^8.17.1" - Update ajv-errors to version "3.0.0" - Update ajv-formats to version "3.0.1" - Update ajv-keywords to version "5.1.0" - Update other dependencies in package-lock.json --- package-lock.json | 151 ++++++++++++++++++++++++++++------------------ package.json | 8 +-- 2 files changed, 95 insertions(+), 64 deletions(-) diff --git a/package-lock.json b/package-lock.json index 173a859..ddd9c38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,42 +13,23 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "file:../common", + "doc-detective-common": "^3.6.0-dev.2", "dotenv": "^17.2.3", "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", - "posthog-node": "^5.17.2" + "posthog-node": "^5.18.1" }, "devDependencies": { "body-parser": "^2.2.1", - "chai": "^6.2.1", + "chai": "^6.2.2", "express": "^5.2.1", "mocha": "^11.7.5", "proxyquire": "^2.1.3", "semver": "^7.7.3", - "sinon": "^21.0.0", + "sinon": "^21.0.1", "yaml": "^2.8.2" } }, - "../common": { - "name": "doc-detective-common", - "version": "3.6.0-dev.1", - "license": "AGPL-3.0-only", - "dependencies": { - "@apidevtools/json-schema-ref-parser": "^15.1.3", - "ajv": "^8.17.1", - "ajv-errors": "^3.0.0", - "ajv-formats": "^3.0.1", - "ajv-keywords": "^5.1.0", - "axios": "^1.13.2", - "yaml": "^2.8.2" - }, - "devDependencies": { - "chai": "^6.2.1", - "mocha": "^11.7.5", - "sinon": "^21.0.0" - } - }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "15.1.3", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-15.1.3.tgz", @@ -145,9 +126,9 @@ } }, "node_modules/@posthog/core": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.7.1.tgz", - "integrity": "sha512-kjK0eFMIpKo9GXIbts8VtAknsoZ18oZorANdtuTj1CbgS28t4ZVq//HAWhnxEuXRTrtkd+SUJ6Ux3j2Af8NCuA==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.9.0.tgz", + "integrity": "sha512-j7KSWxJTUtNyKynLt/p0hfip/3I46dWU2dk+pt7dKRoz2l5CYueHuHK4EO7Wlgno5yo1HO4sc4s30MXMTICHJw==", "license": "MIT", "dependencies": { "cross-spawn": "^7.0.6" @@ -164,9 +145,9 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", - "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.1.0.tgz", + "integrity": "sha512-cqfapCxwTGsrR80FEgOoPsTonoefMBY7dnUEbQ+GRcved0jvkJLzvX6F4WtN+HBqbPX/SiFsIRUp+IrCW/2I2w==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -174,14 +155,13 @@ } }, "node_modules/@sinonjs/samsam": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.2.tgz", - "integrity": "sha512-v46t/fwnhejRSFTGqbpn9u+LQ9xJDse10gNnPgAcxgdoCDMXj/G2asWAC/8Qs+BAZDicX+MNZouXT1A7c83kVw==", + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.3.tgz", + "integrity": "sha512-hw6HbX+GyVZzmaYNh82Ecj1vdGZrqVIn/keDTg63IgAwiQPO+xCz99uG6Woqgb4tM0mUiFENKZ4cqd7IX94AXQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^3.0.1", - "lodash.get": "^4.4.2", "type-detect": "^4.1.0" } }, @@ -252,6 +232,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -263,6 +244,44 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-errors": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-3.0.0.tgz", + "integrity": "sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^8.0.1" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -418,9 +437,9 @@ } }, "node_modules/chai": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.1.tgz", - "integrity": "sha512-p4Z49OGG5W/WBCPSS/dH3jQ73kD6tiMmUM+bckNK6Jr5JHMG3k9bg/BvKR8lKmtVBKmOiuVaV2ws8s9oSbwysg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", "engines": { @@ -694,8 +713,19 @@ } }, "node_modules/doc-detective-common": { - "resolved": "../common", - "link": true + "version": "3.6.0-dev.2", + "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.6.0-dev.2.tgz", + "integrity": "sha512-6wrfWvUSNVh8I1ccqCI7Z0ICxK5j59X2XttLsspcfiSTA42Rsm7jTlFrbMFxQOGBukyVa3mIi44maF9FCTkvdw==", + "license": "AGPL-3.0-only", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^15.1.3", + "ajv": "^8.17.1", + "ajv-errors": "^3.0.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0", + "axios": "^1.13.2", + "yaml": "^2.8.2" + } }, "node_modules/dotenv": { "version": "17.2.3", @@ -1461,14 +1491,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", - "deprecated": "This package is deprecated. Use the optional chaining (?.) operator instead.", - "dev": true, - "license": "MIT" - }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -1789,12 +1811,12 @@ "license": "ISC" }, "node_modules/posthog-node": { - "version": "5.17.2", - "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.17.2.tgz", - "integrity": "sha512-lz3YJOr0Nmiz0yHASaINEDHqoV+0bC3eD8aZAG+Ky292dAnVYul+ga/dMX8KCBXg8hHfKdxw0SztYD5j6dgUqQ==", + "version": "5.18.1", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.18.1.tgz", + "integrity": "sha512-Hi7cRqAlvuEitdiurXJFdMip+BxcwYoX66at5RErMVP91V+Ph9BspGiawC3mJx/4znjwUjF29kAhf8oZQ2uJ5Q==", "license": "MIT", "dependencies": { - "@posthog/core": "1.7.1" + "@posthog/core": "1.9.0" }, "engines": { "node": ">=20" @@ -1832,9 +1854,9 @@ } }, "node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -2173,16 +2195,16 @@ } }, "node_modules/sinon": { - "version": "21.0.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-21.0.0.tgz", - "integrity": "sha512-TOgRcwFPbfGtpqvZw+hyqJDvqfapr1qUlOizROIk4bBLjlsjlB00Pg6wMFXNtJRpu+eCZuVOaLatG7M8105kAw==", + "version": "21.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-21.0.1.tgz", + "integrity": "sha512-Z0NVCW45W8Mg5oC/27/+fCqIHFnW8kpkFOq0j9XJIev4Ld0mKmERaZv5DMLAb9fGCevjKwaEeIQz5+MBXfZcDw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^3.0.1", - "@sinonjs/fake-timers": "^13.0.5", - "@sinonjs/samsam": "^8.0.1", - "diff": "^7.0.0", + "@sinonjs/fake-timers": "^15.1.0", + "@sinonjs/samsam": "^8.0.3", + "diff": "^8.0.2", "supports-color": "^7.2.0" }, "funding": { @@ -2190,6 +2212,16 @@ "url": "https://opencollective.com/sinon" } }, + "node_modules/sinon/node_modules/diff": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.2.tgz", + "integrity": "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/sinon/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2588,7 +2620,6 @@ "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", - "dev": true, "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/package.json b/package.json index 2aa74f4..94533b6 100644 --- a/package.json +++ b/package.json @@ -30,20 +30,20 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "file:../common", + "doc-detective-common": "^3.6.0-dev.2", "dotenv": "^17.2.3", "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", - "posthog-node": "^5.17.2" + "posthog-node": "^5.18.1" }, "devDependencies": { "body-parser": "^2.2.1", - "chai": "^6.2.1", + "chai": "^6.2.2", "express": "^5.2.1", "mocha": "^11.7.5", "proxyquire": "^2.1.3", "semver": "^7.7.3", - "sinon": "^21.0.0", + "sinon": "^21.0.1", "yaml": "^2.8.2" } } From fb53596cb36148052d961d4a5f65a44a239e0a1e Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Sat, 3 Jan 2026 18:18:35 -0800 Subject: [PATCH 9/9] feat: Enhance Heretto integration tests and improve error handling - Update HERETTO_SCENARIO_NAME handling in integration tests to clarify it's optional. - Refine regex patterns for screenshot image handling in config. - Add cleanup logic for temporary directories in Heretto integration tests. - Implement file existence check before uploading to Heretto. --- .github/workflows/integration-tests.yml | 3 ++- src/config.js | 5 ++-- src/heretto.integration.test.js | 31 +++++++++++++++++++++++++ src/heretto.js | 14 ++++++++++- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a372c93..1293867 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -47,7 +47,8 @@ jobs: HERETTO_ORGANIZATION_ID: ${{ secrets.HERETTO_ORGANIZATION_ID }} HERETTO_USERNAME: ${{ secrets.HERETTO_USERNAME }} HERETTO_API_TOKEN: ${{ secrets.HERETTO_API_TOKEN }} - HERETTO_SCENARIO_NAME: ${{ secrets.HERETTO_SCENARIO_NAME || 'Doc Detective' }} + # HERETTO_SCENARIO_NAME is optional - defaults to 'Doc Detective' in the application + HERETTO_SCENARIO_NAME: ${{ secrets.HERETTO_SCENARIO_NAME }} run: npm run test:integration - name: Upload test results diff --git a/src/config.js b/src/config.js index b363a2e..0dc59f3 100644 --- a/src/config.js +++ b/src/config.js @@ -244,10 +244,11 @@ let defaultFileTypes = { { name: "screenshotImage", regex: [ - ']*outputclass="[^"]*screenshot[^"]*href="([^"]+)"[^>]*"[^>]*\\/>', + ']*outputclass="[^"]*screenshot[^"]*"[^>]*href="([^"]+)"[^>]*\\/>', ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[^>]*\\/>', ']*outputclass="[^"]*screenshot[^"]*"[^>]*href="([^"]+)"[\\s\\S]*?<\\/image>', - ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[\\s\\S]*?<\\/image>', ], + ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[\\s\\S]*?<\\/image>', + ], actions: ["screenshot"], }, { diff --git a/src/heretto.integration.test.js b/src/heretto.integration.test.js index a26e987..f3a7919 100644 --- a/src/heretto.integration.test.js +++ b/src/heretto.integration.test.js @@ -15,6 +15,9 @@ */ const heretto = require("./heretto"); +const fs = require("fs"); +const path = require("path"); +const os = require("os"); before(async function () { const { expect } = await import("chai"); @@ -57,6 +60,7 @@ describeIntegration("Heretto Integration Tests (CI Only)", function () { let client; let herettoConfig; + let tempDirectories = []; // Track temp directories for cleanup const mockLog = (...args) => { if (process.env.DEBUG) { console.log(...args); @@ -76,6 +80,33 @@ describeIntegration("Heretto Integration Tests (CI Only)", function () { client = heretto.createApiClient(herettoConfig); }); + after(function () { + // Clean up any temporary directories created during tests + const tempDir = path.join(os.tmpdir(), "doc-detective"); + if (fs.existsSync(tempDir)) { + try { + // Find and remove heretto_* directories created during this test run + const items = fs.readdirSync(tempDir); + for (const item of items) { + if (item.startsWith("heretto_")) { + const itemPath = path.join(tempDir, item); + if (fs.statSync(itemPath).isDirectory()) { + fs.rmSync(itemPath, { recursive: true, force: true }); + if (process.env.DEBUG) { + console.log(`Cleaned up temp directory: ${itemPath}`); + } + } + } + } + } catch (error) { + // Ignore cleanup errors - these are best-effort + if (process.env.DEBUG) { + console.log(`Cleanup warning: ${error.message}`); + } + } + } + }); + describe("API Client Creation", function () { it("should create a valid API client", function () { expect(client).to.not.be.null; diff --git a/src/heretto.js b/src/heretto.js index cedf1a2..be6f7d8 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -570,7 +570,6 @@ async function getResourceDependencies(restClient, ditamapId, log, config) { } catch (error) { // Log more details about the error for debugging const statusCode = error.response?.status; - const responseData = error.response?.data; log(config, "debug", `Dependencies endpoint not available (${statusCode}), will use ditamap info as fallback`); // Continue with ditamap info only - the fallback will create files in the ditamap's parent folder } @@ -908,6 +907,19 @@ async function uploadFile(herettoConfig, fileId, localFilePath, log, config) { const client = createRestApiClient(herettoConfig); try { + // Ensure the local file exists before attempting to read it + if (!fs.existsSync(localFilePath)) { + log( + config, + "warning", + `Local file does not exist, cannot upload to Heretto: ${localFilePath}` + ); + return { + status: "FAIL", + description: `Local file not found: ${localFilePath}`, + }; + } + // Read file as binary const fileBuffer = fs.readFileSync(localFilePath);