From 14ef5de1a65b37de0f2715bde4baf45197168fb0 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 2 Dec 2025 19:43:39 -0800 Subject: [PATCH 01/17] First pass at Heretto integration --- package-lock.json | 10 ++ package.json | 1 + src/config.js | 44 +++++++ src/heretto.js | 288 ++++++++++++++++++++++++++++++++++++++++++++ src/heretto.test.js | 270 +++++++++++++++++++++++++++++++++++++++++ src/utils.js | 10 ++ 6 files changed, 623 insertions(+) create mode 100644 src/heretto.js create mode 100644 src/heretto.test.js diff --git a/package-lock.json b/package-lock.json index b7120d1..c4e9738 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "AGPL-3.0-only", "dependencies": { "@apidevtools/json-schema-ref-parser": "^15.1.3", + "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", "doc-detective-common": "^3.6.0", @@ -217,6 +218,15 @@ "node": ">= 0.6" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", diff --git a/package.json b/package.json index 8e6fb29..a7fd5a0 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "homepage": "https://github.com/doc-detective/doc-detective-core#readme", "dependencies": { "@apidevtools/json-schema-ref-parser": "^15.1.3", + "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", "doc-detective-common": "^3.6.0", diff --git a/src/config.js b/src/config.js index 72eb211..d6c9b01 100644 --- a/src/config.js +++ b/src/config.js @@ -2,6 +2,7 @@ const os = require("os"); const { validate } = require("doc-detective-common"); const { log, loadEnvs, replaceEnvs } = require("./utils"); const { loadDescription } = require("./openapi"); +const { loadHerettoContent } = require("./heretto"); exports.setConfig = setConfig; exports.resolveConcurrentRunners = resolveConcurrentRunners; @@ -640,6 +641,9 @@ async function setConfig({ config }) { // TODO: Revise loadDescriptions() so it doesn't mutate the input but instead returns an updated object await loadDescriptions(config); + + // Load Heretto content from configured integrations + await loadHerettoConfigs(config); return config; } @@ -678,6 +682,46 @@ async function loadDescriptions(config) { } } +/** + * Loads content from all configured Heretto CMS integrations. + * + * @async + * @param {Object} config - The configuration object. + * @returns {Promise} - A promise that resolves when all Heretto content is loaded. + * + * @remarks + * This function modifies the input config object by: + * 1. Adding an 'outputPath' property to each Heretto configuration with the local path to downloaded content. + * 2. Removing any Heretto configurations where the content failed to load. + */ +async function loadHerettoConfigs(config) { + if (config?.integrations?.heretto) { + for (const herettoConfig of config.integrations.heretto) { + try { + const outputPath = await loadHerettoContent(herettoConfig, log, config); + if (outputPath) { + herettoConfig.outputPath = outputPath; + } else { + // Remove the failed Heretto configuration + config.integrations.heretto = config.integrations.heretto.filter( + (item) => item !== herettoConfig + ); + } + } catch (error) { + log( + config, + "warning", + `Failed to load Heretto content from "${herettoConfig.name}": ${error.message}` + ); + // Remove the failed Heretto configuration + config.integrations.heretto = config.integrations.heretto.filter( + (item) => item !== herettoConfig + ); + } + } + } +} + // Detect aspects of the environment running Doc Detective. function getEnvironment() { const environment = {}; diff --git a/src/heretto.js b/src/heretto.js new file mode 100644 index 0000000..c547a0f --- /dev/null +++ b/src/heretto.js @@ -0,0 +1,288 @@ +const axios = require("axios"); +const fs = require("fs"); +const path = require("path"); +const os = require("os"); +const crypto = require("crypto"); +const AdmZip = require("adm-zip"); + +// Internal constants - not exposed to users +const POLLING_INTERVAL_MS = 5000; +const POLLING_TIMEOUT_MS = 300000; // 5 minutes +const SCENARIO_NAME = "Doc Detective"; +const SCENARIO_DESCRIPTION = "Normalized DITA output for Doc Detective testing"; + +/** + * Creates a Base64-encoded Basic Auth header from username and API token. + * @param {string} username - Heretto CCMS username (email) + * @param {string} apiToken - API token generated in Heretto CCMS + * @returns {string} Base64-encoded authorization header value + */ +function createAuthHeader(username, apiToken) { + const credentials = `${username}:${apiToken}`; + return Buffer.from(credentials).toString("base64"); +} + +/** + * Builds the base URL for Heretto CCMS API. + * @param {string} organizationId - The organization subdomain + * @returns {string} Base API URL + */ +function getBaseUrl(organizationId) { + return `https://${organizationId}.heretto.com/ezdnxtgen/api/v2`; +} + +/** + * Creates an axios instance configured for Heretto API requests. + * @param {Object} herettoConfig - Heretto integration configuration + * @returns {Object} Configured axios instance + */ +function createApiClient(herettoConfig) { + const authHeader = createAuthHeader( + herettoConfig.username, + herettoConfig.apiToken + ); + return axios.create({ + baseURL: getBaseUrl(herettoConfig.organizationId), + headers: { + Authorization: `Basic ${authHeader}`, + "Content-Type": "application/json", + }, + }); +} + +/** + * Fetches all available publishing scenarios from Heretto. + * @param {Object} client - Configured axios instance + * @returns {Promise} Array of publishing scenarios + */ +async function getPublishingScenarios(client) { + const response = await client.get("/publishing-scenarios"); + return response.data.content || []; +} + +/** + * Creates a new publishing scenario in Heretto. + * @param {Object} client - Configured axios instance + * @returns {Promise} Created scenario object + */ +async function createPublishingScenario(client) { + const scenarioConfig = { + name: SCENARIO_NAME, + description: SCENARIO_DESCRIPTION, + outputFormat: "dita", + engine: "dita-ot", + parameters: { + "transtype": "dita" + } + }; + const response = await client.post("/publishing-scenarios", scenarioConfig); + return response.data; +} + +/** + * Finds an existing "Doc Detective" scenario or creates one if missing. + * @param {Object} client - Configured axios instance + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Scenario object or null if creation failed + */ +async function findOrCreateScenario(client, log, config) { + try { + const scenarios = await getPublishingScenarios(client); + const existingScenario = scenarios.find((s) => s.name === SCENARIO_NAME); + + if (existingScenario) { + log(config, "debug", `Found existing "${SCENARIO_NAME}" scenario: ${existingScenario.id}`); + return existingScenario; + } + + log(config, "info", `Creating "${SCENARIO_NAME}" publishing scenario...`); + const newScenario = await createPublishingScenario(client); + log(config, "debug", `Created scenario: ${newScenario.id}`); + return newScenario; + } catch (error) { + log( + config, + "error", + `Failed to find or create publishing scenario: ${error.message}` + ); + return null; + } +} + +/** + * Triggers a publishing job for a DITA map. + * @param {Object} client - Configured axios instance + * @param {string} fileId - UUID of the DITA map + * @param {string} scenarioId - ID of the publishing scenario to use + * @returns {Promise} Publishing job object + */ +async function triggerPublishingJob(client, fileId, scenarioId) { + const response = await client.post(`/files/${fileId}/publishing-jobs`, { + scenarioId: scenarioId, + }); + return response.data; +} + +/** + * Gets the status of a publishing job. + * @param {Object} client - Configured axios instance + * @param {string} fileId - UUID of the DITA map + * @param {string} jobId - ID of the publishing job + * @returns {Promise} Job status object + */ +async function getJobStatus(client, fileId, jobId) { + const response = await client.get(`/files/${fileId}/publishing-jobs/${jobId}`); + return response.data; +} + +/** + * Polls a publishing job until completion or timeout. + * @param {Object} client - Configured axios instance + * @param {string} fileId - UUID of the DITA map + * @param {string} jobId - ID of the publishing job + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Completed job object or null on timeout/failure + */ +async function pollJobStatus(client, fileId, jobId, log, config) { + const startTime = Date.now(); + + while (Date.now() - startTime < POLLING_TIMEOUT_MS) { + try { + const job = await getJobStatus(client, fileId, jobId); + log(config, "debug", `Job ${jobId} status: ${job.status}`); + + if (job.status === "COMPLETED" || job.status === "SUCCESS") { + return job; + } + + if (job.status === "FAILED" || job.status === "ERROR") { + log(config, "warning", `Publishing job ${jobId} failed: ${job.errorMessage || "Unknown error"}`); + return null; + } + + // Wait before next poll + await new Promise((resolve) => setTimeout(resolve, POLLING_INTERVAL_MS)); + } catch (error) { + log(config, "warning", `Error polling job status: ${error.message}`); + return null; + } + } + + log(config, "warning", `Publishing job ${jobId} timed out after ${POLLING_TIMEOUT_MS / 1000} seconds`); + return null; +} + +/** + * Downloads the publishing job output and extracts it to temp directory. + * @param {Object} client - Configured axios instance + * @param {string} fileId - UUID of the DITA map + * @param {string} jobId - ID of the publishing job + * @param {string} herettoName - Name of the Heretto integration for directory naming + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Path to extracted content or null on failure + */ +async function downloadAndExtractOutput(client, fileId, jobId, herettoName, log, config) { + try { + // Create temp directory if it doesn't exist + const tempDir = `${os.tmpdir()}/doc-detective`; + if (!fs.existsSync(tempDir)) { + fs.mkdirSync(tempDir, { recursive: true }); + } + + // Create unique output directory based on heretto name and job ID + const hash = crypto.createHash("md5").update(`${herettoName}_${jobId}`).digest("hex"); + const outputDir = path.join(tempDir, `heretto_${hash}`); + + // Download the output file + log(config, "debug", `Downloading publishing job output for ${herettoName}...`); + const response = await client.get(`/files/${fileId}/publishing-jobs/${jobId}/output`, { + responseType: "arraybuffer", + }); + + // Save ZIP to temp file + const zipPath = path.join(tempDir, `heretto_${hash}.zip`); + fs.writeFileSync(zipPath, response.data); + + // Extract ZIP contents + log(config, "debug", `Extracting output to ${outputDir}...`); + const zip = new AdmZip(zipPath); + zip.extractAllTo(outputDir, true); + + // Clean up ZIP file + fs.unlinkSync(zipPath); + + log(config, "info", `Heretto content "${herettoName}" extracted to ${outputDir}`); + return outputDir; + } catch (error) { + log(config, "warning", `Failed to download or extract output: ${error.message}`); + return null; + } +} + +/** + * Main function to load content from a Heretto CMS instance. + * Triggers a publishing job, waits for completion, and downloads the output. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Path to extracted content or null on failure + */ +async function loadHerettoContent(herettoConfig, log, config) { + log(config, "info", `Loading content from Heretto "${herettoConfig.name}"...`); + + try { + const client = createApiClient(herettoConfig); + + // Find or create the Doc Detective publishing scenario + const scenario = await findOrCreateScenario(client, log, config); + if (!scenario) { + log(config, "warning", `Skipping Heretto "${herettoConfig.name}" - could not find or create publishing scenario`); + return null; + } + + // Trigger publishing job + log(config, "debug", `Triggering publishing job for file ${herettoConfig.fileId}...`); + const job = await triggerPublishingJob(client, herettoConfig.fileId, scenario.id); + log(config, "debug", `Publishing job started: ${job.id}`); + + // Poll for completion + log(config, "info", `Waiting for publishing job to complete...`); + const completedJob = await pollJobStatus(client, herettoConfig.fileId, job.id, log, config); + if (!completedJob) { + log(config, "warning", `Skipping Heretto "${herettoConfig.name}" - publishing job failed or timed out`); + return null; + } + + // Download and extract output + const outputPath = await downloadAndExtractOutput( + client, + herettoConfig.fileId, + completedJob.id, + herettoConfig.name, + log, + config + ); + + return outputPath; + } catch (error) { + log(config, "warning", `Failed to load Heretto "${herettoConfig.name}": ${error.message}`); + return null; + } +} + +module.exports = { + createAuthHeader, + createApiClient, + findOrCreateScenario, + triggerPublishingJob, + pollJobStatus, + downloadAndExtractOutput, + loadHerettoContent, + // Export constants for testing + POLLING_INTERVAL_MS, + POLLING_TIMEOUT_MS, + SCENARIO_NAME, +}; diff --git a/src/heretto.test.js b/src/heretto.test.js new file mode 100644 index 0000000..3756c01 --- /dev/null +++ b/src/heretto.test.js @@ -0,0 +1,270 @@ +const assert = require("assert"); +const sinon = require("sinon"); +const proxyquire = require("proxyquire"); +const path = require("path"); +const fs = require("fs"); +const os = require("os"); + +before(async function () { + const { expect } = await import("chai"); + global.expect = expect; +}); + +describe("Heretto Integration", function () { + let heretto; + let axiosStub; + let axiosCreateStub; + let mockClient; + + beforeEach(function () { + // Create mock axios client + mockClient = { + get: sinon.stub(), + post: sinon.stub(), + }; + + // Stub axios.create to return our mock client + axiosCreateStub = sinon.stub().returns(mockClient); + + // Use proxyquire to inject stubbed axios + heretto = proxyquire("../src/heretto", { + axios: { + create: axiosCreateStub, + }, + }); + }); + + afterEach(function () { + sinon.restore(); + }); + + describe("createAuthHeader", function () { + it("should create a Base64-encoded auth header", function () { + const authHeader = heretto.createAuthHeader("user@example.com", "token123"); + + // Base64 of "user@example.com:token123" + const expected = Buffer.from("user@example.com:token123").toString("base64"); + expect(authHeader).to.equal(expected); + }); + + it("should handle special characters in credentials", function () { + const authHeader = heretto.createAuthHeader("user@example.com", "p@ss:w0rd!"); + + const expected = Buffer.from("user@example.com:p@ss:w0rd!").toString("base64"); + expect(authHeader).to.equal(expected); + }); + }); + + describe("createApiClient", function () { + it("should create an axios client with correct config", function () { + const herettoConfig = { + organizationId: "thunderbird", + username: "user@example.com", + apiToken: "token123", + }; + + heretto.createApiClient(herettoConfig); + + expect(axiosCreateStub.calledOnce).to.be.true; + const createConfig = axiosCreateStub.firstCall.args[0]; + expect(createConfig.baseURL).to.equal("https://thunderbird.heretto.com/ezdnxtgen/api/v2"); + expect(createConfig.headers.Authorization).to.include("Basic "); + expect(createConfig.headers["Content-Type"]).to.equal("application/json"); + }); + }); + + describe("findOrCreateScenario", function () { + const mockLog = sinon.stub(); + const mockConfig = { logLevel: "info" }; + + beforeEach(function () { + mockLog.reset(); + }); + + it("should return existing scenario if found", async function () { + const existingScenario = { + id: "scenario-123", + name: "Doc Detective", + }; + + mockClient.get.resolves({ + data: { content: [existingScenario, { id: "other", name: "Other" }] }, + }); + + const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + + expect(result).to.deep.equal(existingScenario); + expect(mockClient.get.calledOnce).to.be.true; + expect(mockClient.post.called).to.be.false; + }); + + it("should create scenario if not found", async function () { + const newScenario = { + id: "new-scenario-123", + name: "Doc Detective", + }; + + mockClient.get.resolves({ + data: { content: [{ id: "other", name: "Other Scenario" }] }, + }); + mockClient.post.resolves({ data: newScenario }); + + const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + + expect(result).to.deep.equal(newScenario); + expect(mockClient.get.calledOnce).to.be.true; + expect(mockClient.post.calledOnce).to.be.true; + }); + + it("should return null if scenario fetch fails", async function () { + mockClient.get.rejects(new Error("Network error")); + + const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + + expect(result).to.be.null; + }); + + it("should return null if scenario creation fails", async function () { + mockClient.get.resolves({ data: { content: [] } }); + mockClient.post.rejects(new Error("Permission denied")); + + const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + + expect(result).to.be.null; + }); + }); + + describe("triggerPublishingJob", function () { + it("should trigger a publishing job", async function () { + const expectedJob = { + id: "job-123", + status: "PENDING", + }; + + mockClient.post.resolves({ data: expectedJob }); + + const result = await heretto.triggerPublishingJob(mockClient, "file-uuid", "scenario-id"); + + expect(result).to.deep.equal(expectedJob); + expect(mockClient.post.calledOnce).to.be.true; + expect(mockClient.post.firstCall.args[0]).to.equal("/files/file-uuid/publishing-jobs"); + expect(mockClient.post.firstCall.args[1]).to.deep.equal({ scenarioId: "scenario-id" }); + }); + }); + + describe("pollJobStatus", function () { + const mockLog = sinon.stub(); + const mockConfig = { logLevel: "info" }; + + beforeEach(function () { + mockLog.reset(); + }); + + it("should return completed job", async function () { + const completedJob = { + id: "job-123", + status: "COMPLETED", + }; + + mockClient.get.resolves({ data: completedJob }); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.deep.equal(completedJob); + }); + + it("should return null for failed job", async function () { + const failedJob = { + id: "job-123", + status: "FAILED", + errorMessage: "Build failed", + }; + + mockClient.get.resolves({ data: failedJob }); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.be.null; + }); + + it("should poll until completion", async function () { + // Use fake timers to avoid waiting for real POLLING_INTERVAL_MS delays + const clock = sinon.useFakeTimers(); + + mockClient.get + .onFirstCall().resolves({ data: { id: "job-123", status: "PENDING" } }) + .onSecondCall().resolves({ data: { id: "job-123", status: "PROCESSING" } }) + .onThirdCall().resolves({ data: { id: "job-123", status: "COMPLETED" } }); + + const pollPromise = heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + // Advance time past the polling intervals + await clock.tickAsync(heretto.POLLING_INTERVAL_MS); + await clock.tickAsync(heretto.POLLING_INTERVAL_MS); + await clock.tickAsync(heretto.POLLING_INTERVAL_MS); + + const result = await pollPromise; + + expect(result.status).to.equal("COMPLETED"); + expect(mockClient.get.callCount).to.equal(3); + + clock.restore(); + }); + }); + + describe("loadHerettoContent", function () { + const mockLog = sinon.stub(); + const mockConfig = { logLevel: "info" }; + + beforeEach(function () { + mockLog.reset(); + }); + + it("should return null if scenario creation fails", async function () { + const herettoConfig = { + name: "test-heretto", + organizationId: "test-org", + username: "user@example.com", + apiToken: "token123", + fileId: "file-uuid", + }; + + // Scenario fetch fails + mockClient.get.rejects(new Error("Network error")); + + const result = await heretto.loadHerettoContent(herettoConfig, mockLog, mockConfig); + + expect(result).to.be.null; + }); + + it("should return null if publishing job fails", async function () { + const herettoConfig = { + name: "test-heretto", + organizationId: "test-org", + username: "user@example.com", + apiToken: "token123", + fileId: "file-uuid", + }; + + // Scenario exists + mockClient.get.onFirstCall().resolves({ + data: { content: [{ id: "scenario-123", name: "Doc Detective" }] }, + }); + + // Job creation fails + mockClient.post.rejects(new Error("Job creation failed")); + + const result = await heretto.loadHerettoContent(herettoConfig, mockLog, mockConfig); + + expect(result).to.be.null; + }); + }); + + describe("Constants", function () { + it("should export expected constants", function () { + expect(heretto.POLLING_INTERVAL_MS).to.equal(5000); + expect(heretto.POLLING_TIMEOUT_MS).to.equal(300000); + expect(heretto.SCENARIO_NAME).to.equal("Doc Detective"); + }); + }); +}); diff --git a/src/utils.js b/src/utils.js index 955b232..0f2cb1b 100644 --- a/src/utils.js +++ b/src/utils.js @@ -192,6 +192,16 @@ async function qualifyFiles({ config }) { let sequence = []; // Determine source sequence + // Add Heretto output paths to the beginning of the sequence + if (config?.integrations?.heretto) { + for (const herettoConfig of config.integrations.heretto) { + if (herettoConfig.outputPath) { + log(config, "debug", `Adding Heretto output path: ${herettoConfig.outputPath}`); + sequence.unshift(herettoConfig.outputPath); + } + } + } + const setup = config.beforeAny; if (setup) sequence = sequence.concat(setup); const input = config.input; From c43daf537fe7121b95162a298815a62e1652ad0a Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Thu, 4 Dec 2025 17:13:39 -0800 Subject: [PATCH 02/17] Fix resolver log level --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 4586e31..34ece77 100644 --- a/src/index.js +++ b/src/index.js @@ -34,7 +34,7 @@ async function detectAndResolveTests({ config }) { // Detect tests const detectedTests = await detectTests({ config }); if (!detectedTests || detectedTests.length === 0) { - log(config, "warn", "No tests detected."); + log(config, "warning", "No tests detected."); return null; } // Resolve tests From f419c4e1d0518ef4a9ac2ba3a14f218cb6601d3e Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Thu, 4 Dec 2025 19:15:16 -0800 Subject: [PATCH 03/17] Heretto API failure workflows --- package-lock.json | 75 +++++---------- package.json | 2 +- src/config.js | 9 -- src/heretto.js | 223 ++++++++++++++++++++++++++++++++++++-------- src/heretto.test.js | 10 +- src/utils.js | 5 + 6 files changed, 217 insertions(+), 107 deletions(-) diff --git a/package-lock.json b/package-lock.json index c4e9738..6a28780 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "^3.6.0", + "doc-detective-common": "file:../common", "dotenv": "^17.2.3", "json-schema-faker": "^0.5.9", "posthog-node": "^5.15.0" @@ -29,6 +29,25 @@ "yaml": "^2.8.2" } }, + "../common": { + "name": "doc-detective-common", + "version": "3.6.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^15.1.3", + "ajv": "^8.17.1", + "ajv-errors": "^3.0.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0", + "axios": "^1.13.2", + "yaml": "^2.8.2" + }, + "devDependencies": { + "chai": "^6.2.1", + "mocha": "^11.7.5", + "sinon": "^21.0.0" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "15.1.3", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-15.1.3.tgz", @@ -243,44 +262,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ajv-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-3.0.0.tgz", - "integrity": "sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==", - "license": "MIT", - "peerDependencies": { - "ajv": "^8.0.1" - } - }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ajv-keywords": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", - "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3" - }, - "peerDependencies": { - "ajv": "^8.8.2" - } - }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -712,19 +693,8 @@ } }, "node_modules/doc-detective-common": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.6.0.tgz", - "integrity": "sha512-rHZ3WNuw3Y51KduBh9GgnrYFzJdbCQHPP3xnkp+W8QyF+4bKhWFlVVmbotFh/XL3BupbV9+CP7YEBTsW02Hvuw==", - "license": "AGPL-3.0-only", - "dependencies": { - "@apidevtools/json-schema-ref-parser": "^15.1.3", - "ajv": "^8.17.1", - "ajv-errors": "^3.0.0", - "ajv-formats": "^3.0.1", - "ajv-keywords": "^5.1.0", - "axios": "^1.13.2", - "yaml": "^2.8.2" - } + "resolved": "../common", + "link": true }, "node_modules/dotenv": { "version": "17.2.3", @@ -2586,6 +2556,7 @@ "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "dev": true, "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/package.json b/package.json index a7fd5a0..f0fa383 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "^3.6.0", + "doc-detective-common": "file:../common", "dotenv": "^17.2.3", "json-schema-faker": "^0.5.9", "posthog-node": "^5.15.0" diff --git a/src/config.js b/src/config.js index d6c9b01..4d1e129 100644 --- a/src/config.js +++ b/src/config.js @@ -701,11 +701,6 @@ async function loadHerettoConfigs(config) { const outputPath = await loadHerettoContent(herettoConfig, log, config); if (outputPath) { herettoConfig.outputPath = outputPath; - } else { - // Remove the failed Heretto configuration - config.integrations.heretto = config.integrations.heretto.filter( - (item) => item !== herettoConfig - ); } } catch (error) { log( @@ -713,10 +708,6 @@ async function loadHerettoConfigs(config) { "warning", `Failed to load Heretto content from "${herettoConfig.name}": ${error.message}` ); - // Remove the failed Heretto configuration - config.integrations.heretto = config.integrations.heretto.filter( - (item) => item !== herettoConfig - ); } } } diff --git a/src/heretto.js b/src/heretto.js index c547a0f..b83c9de 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -56,10 +56,17 @@ function createApiClient(herettoConfig) { * @returns {Promise} Array of publishing scenarios */ async function getPublishingScenarios(client) { - const response = await client.get("/publishing-scenarios"); + const response = await client.get("/publishes/scenarios"); return response.data.content || []; } +async function getPublishingScenarioParameters(client, scenarioId) { + const response = await client.get( + `/publishes/scenarios/${scenarioId}/parameters` + ); + return response.data; +} + /** * Creates a new publishing scenario in Heretto. * @param {Object} client - Configured axios instance @@ -72,13 +79,14 @@ async function createPublishingScenario(client) { outputFormat: "dita", engine: "dita-ot", parameters: { - "transtype": "dita" - } + transtype: "dita", + }, }; const response = await client.post("/publishing-scenarios", scenarioConfig); return response.data; } +// TODO: Remove scenario creation. Users have to create it manually with specific settings. /** * Finds an existing "Doc Detective" scenario or creates one if missing. * @param {Object} client - Configured axios instance @@ -86,25 +94,80 @@ async function createPublishingScenario(client) { * @param {Object} config - Doc Detective config for logging * @returns {Promise} Scenario object or null if creation failed */ -async function findOrCreateScenario(client, log, config) { +async function findScenario(client, log, config) { try { const scenarios = await getPublishingScenarios(client); - const existingScenario = scenarios.find((s) => s.name === SCENARIO_NAME); + const foundScenario = scenarios.find((s) => s.name === SCENARIO_NAME); + + if (!foundScenario) { + log(config, "error", `No existing "${SCENARIO_NAME}" scenario found.`); + return null; + } + + const scenarioParameters = await getPublishingScenarioParameters( + client, + foundScenario.id + ); + + if (!scenarioParameters) { + log( + config, + "error", + `Failed to retrieve scenario details for ID: ${foundScenario.id}` + ); + return null; + } - if (existingScenario) { - log(config, "debug", `Found existing "${SCENARIO_NAME}" scenario: ${existingScenario.id}`); - return existingScenario; + // Make sure that scenarioParameters.content has an object with name="transtype" and options[0].value="dita" + const transtypeParam = scenarioParameters.content.find( + (param) => param.name === "transtype" + ); + if (!transtypeParam || !transtypeParam.options || transtypeParam.options[0].value !== "dita") { + log( + config, + "error", + `Existing "${SCENARIO_NAME}" scenario has incorrect "transtype" parameter settings. Make sure it is set to "dita".` + ); + return null; + } + + // Make sure that senarioParameters.content has an object with name="tool-kit-name" and value="default/dita-ot-3.6.1" + const toolKitParam = scenarioParameters.content.find( + (param) => param.name === "tool-kit-name" + ); + if (!toolKitParam || !toolKitParam.value || toolKitParam.value !== "default/dita-ot-3.6.1") { + log( + config, + "error", + `Existing "${SCENARIO_NAME}" scenario has incorrect "tool-kit-name" parameter settings. Make sure it is set to "default/dita-ot-3.6.1".` + ); + return null; + } + + // Make sure that scenarioParameters.content has an object with type="file_uuid_picker" and a value + const fileUuidPickerParam = scenarioParameters.content.find( + (param) => param.type === "file_uuid_picker" + ); + if (!fileUuidPickerParam || !fileUuidPickerParam.value) { + log( + config, + "error", + `Existing "${SCENARIO_NAME}" scenario has incorrect "file_uuid_picker" parameter settings. Make sure it has a valid value.` + ); + return null; } - log(config, "info", `Creating "${SCENARIO_NAME}" publishing scenario...`); - const newScenario = await createPublishingScenario(client); - log(config, "debug", `Created scenario: ${newScenario.id}`); - return newScenario; + log( + config, + "debug", + `Found existing "${SCENARIO_NAME}" scenario: ${foundScenario.id}` + ); + return { scenarioId: foundScenario.id, fileId: fileUuidPickerParam.value }; } catch (error) { log( config, "error", - `Failed to find or create publishing scenario: ${error.message}` + `Failed to find publishing scenario: ${error.message}` ); return null; } @@ -118,8 +181,9 @@ async function findOrCreateScenario(client, log, config) { * @returns {Promise} Publishing job object */ async function triggerPublishingJob(client, fileId, scenarioId) { - const response = await client.post(`/files/${fileId}/publishing-jobs`, { - scenarioId: scenarioId, + const response = await client.post(`/files/${fileId}/publishes`, { + scenario: scenarioId, + parameters: [] }); return response.data; } @@ -132,10 +196,17 @@ async function triggerPublishingJob(client, fileId, scenarioId) { * @returns {Promise} Job status object */ async function getJobStatus(client, fileId, jobId) { - const response = await client.get(`/files/${fileId}/publishing-jobs/${jobId}`); + const response = await client.get( + `/files/${fileId}/publishes/${jobId}` + ); return response.data; } +async function getPublishingJobHistory(client, fileId) { + const response = await client.get(`/files/${fileId}/publishes`); + return response.data.content || []; +} + /** * Polls a publishing job until completion or timeout. * @param {Object} client - Configured axios instance @@ -151,14 +222,18 @@ async function pollJobStatus(client, fileId, jobId, log, config) { while (Date.now() - startTime < POLLING_TIMEOUT_MS) { try { const job = await getJobStatus(client, fileId, jobId); - log(config, "debug", `Job ${jobId} status: ${job.status}`); + log(config, "debug", `Job ${jobId} status: ${job?.status?.status}`); - if (job.status === "COMPLETED" || job.status === "SUCCESS") { + if (job?.status?.result === "SUCCESS" ) { return job; } - if (job.status === "FAILED" || job.status === "ERROR") { - log(config, "warning", `Publishing job ${jobId} failed: ${job.errorMessage || "Unknown error"}`); + if (job?.status?.result === "FAIL") { + log( + config, + "warning", + `Publishing job ${jobId} failed.` + ); return null; } @@ -170,7 +245,13 @@ async function pollJobStatus(client, fileId, jobId, log, config) { } } - log(config, "warning", `Publishing job ${jobId} timed out after ${POLLING_TIMEOUT_MS / 1000} seconds`); + log( + config, + "warning", + `Publishing job ${jobId} timed out after ${ + POLLING_TIMEOUT_MS / 1000 + } seconds` + ); return null; } @@ -184,7 +265,14 @@ async function pollJobStatus(client, fileId, jobId, log, config) { * @param {Object} config - Doc Detective config for logging * @returns {Promise} Path to extracted content or null on failure */ -async function downloadAndExtractOutput(client, fileId, jobId, herettoName, log, config) { +async function downloadAndExtractOutput( + client, + fileId, + jobId, + herettoName, + log, + config +) { try { // Create temp directory if it doesn't exist const tempDir = `${os.tmpdir()}/doc-detective`; @@ -193,14 +281,24 @@ async function downloadAndExtractOutput(client, fileId, jobId, herettoName, log, } // Create unique output directory based on heretto name and job ID - const hash = crypto.createHash("md5").update(`${herettoName}_${jobId}`).digest("hex"); + const hash = crypto + .createHash("md5") + .update(`${herettoName}_${jobId}`) + .digest("hex"); const outputDir = path.join(tempDir, `heretto_${hash}`); // Download the output file - log(config, "debug", `Downloading publishing job output for ${herettoName}...`); - const response = await client.get(`/files/${fileId}/publishing-jobs/${jobId}/output`, { - responseType: "arraybuffer", - }); + log( + config, + "debug", + `Downloading publishing job output for ${herettoName}...` + ); + const response = await client.get( + `/files/${fileId}/publishing-jobs/${jobId}/output`, + { + responseType: "arraybuffer", + } + ); // Save ZIP to temp file const zipPath = path.join(tempDir, `heretto_${hash}.zip`); @@ -214,10 +312,18 @@ async function downloadAndExtractOutput(client, fileId, jobId, herettoName, log, // Clean up ZIP file fs.unlinkSync(zipPath); - log(config, "info", `Heretto content "${herettoName}" extracted to ${outputDir}`); + log( + config, + "info", + `Heretto content "${herettoName}" extracted to ${outputDir}` + ); return outputDir; } catch (error) { - log(config, "warning", `Failed to download or extract output: ${error.message}`); + log( + config, + "warning", + `Failed to download or extract output: ${error.message}` + ); return null; } } @@ -231,35 +337,68 @@ async function downloadAndExtractOutput(client, fileId, jobId, herettoName, log, * @returns {Promise} Path to extracted content or null on failure */ async function loadHerettoContent(herettoConfig, log, config) { - log(config, "info", `Loading content from Heretto "${herettoConfig.name}"...`); + log( + config, + "info", + `Loading content from Heretto "${herettoConfig.name}"...` + ); try { const client = createApiClient(herettoConfig); // Find or create the Doc Detective publishing scenario - const scenario = await findOrCreateScenario(client, log, config); + const scenario = await findScenario(client, log, config); if (!scenario) { - log(config, "warning", `Skipping Heretto "${herettoConfig.name}" - could not find or create publishing scenario`); + log( + config, + "warning", + `Skipping Heretto "${herettoConfig.name}" - could not find or create publishing scenario` + ); return null; } + // const history = await getPublishingJobHistory(client, scenario.fileId); + // log( + // config, + // "debug", + // `Found ${history.length} previous publishing jobs for file ${scenario.fileId}` + // ); + // Trigger publishing job - log(config, "debug", `Triggering publishing job for file ${herettoConfig.fileId}...`); - const job = await triggerPublishingJob(client, herettoConfig.fileId, scenario.id); - log(config, "debug", `Publishing job started: ${job.id}`); + log( + config, + "debug", + `Triggering publishing job for file ${scenario.fileId}...` + ); + const job = await triggerPublishingJob( + client, + scenario.fileId, + scenario.scenarioId + ); + log(config, "debug", `Publishing job started: ${job.jobId}`); // Poll for completion log(config, "info", `Waiting for publishing job to complete...`); - const completedJob = await pollJobStatus(client, herettoConfig.fileId, job.id, log, config); + const completedJob = await pollJobStatus( + client, + scenario.fileId, + job.jobId, + log, + config + ); if (!completedJob) { - log(config, "warning", `Skipping Heretto "${herettoConfig.name}" - publishing job failed or timed out`); + log( + config, + "warning", + `Skipping Heretto "${herettoConfig.name}" - publishing job failed or timed out` + ); return null; } // Download and extract output const outputPath = await downloadAndExtractOutput( client, - herettoConfig.fileId, + scenario.fileId, completedJob.id, herettoConfig.name, log, @@ -268,7 +407,11 @@ async function loadHerettoContent(herettoConfig, log, config) { return outputPath; } catch (error) { - log(config, "warning", `Failed to load Heretto "${herettoConfig.name}": ${error.message}`); + log( + config, + "warning", + `Failed to load Heretto "${herettoConfig.name}": ${error}` + ); return null; } } @@ -276,7 +419,7 @@ async function loadHerettoContent(herettoConfig, log, config) { module.exports = { createAuthHeader, createApiClient, - findOrCreateScenario, + findScenario, triggerPublishingJob, pollJobStatus, downloadAndExtractOutput, diff --git a/src/heretto.test.js b/src/heretto.test.js index 3756c01..e646511 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -73,7 +73,7 @@ describe("Heretto Integration", function () { }); }); - describe("findOrCreateScenario", function () { + describe("findScenario", function () { const mockLog = sinon.stub(); const mockConfig = { logLevel: "info" }; @@ -91,7 +91,7 @@ describe("Heretto Integration", function () { data: { content: [existingScenario, { id: "other", name: "Other" }] }, }); - const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig); expect(result).to.deep.equal(existingScenario); expect(mockClient.get.calledOnce).to.be.true; @@ -109,7 +109,7 @@ describe("Heretto Integration", function () { }); mockClient.post.resolves({ data: newScenario }); - const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig); expect(result).to.deep.equal(newScenario); expect(mockClient.get.calledOnce).to.be.true; @@ -119,7 +119,7 @@ describe("Heretto Integration", function () { it("should return null if scenario fetch fails", async function () { mockClient.get.rejects(new Error("Network error")); - const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig); expect(result).to.be.null; }); @@ -128,7 +128,7 @@ describe("Heretto Integration", function () { mockClient.get.resolves({ data: { content: [] } }); mockClient.post.rejects(new Error("Permission denied")); - const result = await heretto.findOrCreateScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig); expect(result).to.be.null; }); diff --git a/src/utils.js b/src/utils.js index 0f2cb1b..f693702 100644 --- a/src/utils.js +++ b/src/utils.js @@ -209,6 +209,11 @@ async function qualifyFiles({ config }) { const cleanup = config.afterAll; if (cleanup) sequence = sequence.concat(cleanup); + if (sequence.length === 0) { + log(config, "warning", "No input sources specified."); + return []; + } + for (let source of sequence) { log(config, "debug", `source: ${source}`); // Check if source is a URL From 2fd032f1e5d9c4c6934285e466bef34204e63305 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Thu, 4 Dec 2025 19:25:12 -0800 Subject: [PATCH 04/17] Set octect-stream accept header --- src/heretto.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/heretto.js b/src/heretto.js index b83c9de..ef29e3f 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -294,9 +294,12 @@ async function downloadAndExtractOutput( `Downloading publishing job output for ${herettoName}...` ); const response = await client.get( - `/files/${fileId}/publishing-jobs/${jobId}/output`, + `/files/${fileId}/publishes/${jobId}/assets-all`, { responseType: "arraybuffer", + headers: { + Accept: "application/octet-stream", + }, } ); From c1f0200cc591d9d8180526f2c2abe4e5141624a9 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 5 Dec 2025 00:43:42 -0800 Subject: [PATCH 05/17] Move Heretto loading into qualifyFiles() --- src/config.js | 35 ----------------------------- src/utils.js | 62 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/config.js b/src/config.js index 4d1e129..72eb211 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,6 @@ const os = require("os"); const { validate } = require("doc-detective-common"); const { log, loadEnvs, replaceEnvs } = require("./utils"); const { loadDescription } = require("./openapi"); -const { loadHerettoContent } = require("./heretto"); exports.setConfig = setConfig; exports.resolveConcurrentRunners = resolveConcurrentRunners; @@ -641,9 +640,6 @@ async function setConfig({ config }) { // TODO: Revise loadDescriptions() so it doesn't mutate the input but instead returns an updated object await loadDescriptions(config); - - // Load Heretto content from configured integrations - await loadHerettoConfigs(config); return config; } @@ -682,37 +678,6 @@ async function loadDescriptions(config) { } } -/** - * Loads content from all configured Heretto CMS integrations. - * - * @async - * @param {Object} config - The configuration object. - * @returns {Promise} - A promise that resolves when all Heretto content is loaded. - * - * @remarks - * This function modifies the input config object by: - * 1. Adding an 'outputPath' property to each Heretto configuration with the local path to downloaded content. - * 2. Removing any Heretto configurations where the content failed to load. - */ -async function loadHerettoConfigs(config) { - if (config?.integrations?.heretto) { - for (const herettoConfig of config.integrations.heretto) { - try { - const outputPath = await loadHerettoContent(herettoConfig, log, config); - if (outputPath) { - herettoConfig.outputPath = outputPath; - } - } catch (error) { - log( - config, - "warning", - `Failed to load Heretto content from "${herettoConfig.name}": ${error.message}` - ); - } - } - } -} - // Detect aspects of the environment running Doc Detective. function getEnvironment() { const environment = {}; diff --git a/src/utils.js b/src/utils.js index f693702..e8a5e89 100644 --- a/src/utils.js +++ b/src/utils.js @@ -11,6 +11,7 @@ const { transformToSchemaKey, readFile, } = require("doc-detective-common"); +const { loadHerettoContent } = require("./heretto"); exports.qualifyFiles = qualifyFiles; exports.parseTests = parseTests; @@ -192,16 +193,6 @@ async function qualifyFiles({ config }) { let sequence = []; // Determine source sequence - // Add Heretto output paths to the beginning of the sequence - if (config?.integrations?.heretto) { - for (const herettoConfig of config.integrations.heretto) { - if (herettoConfig.outputPath) { - log(config, "debug", `Adding Heretto output path: ${herettoConfig.outputPath}`); - sequence.unshift(herettoConfig.outputPath); - } - } - } - const setup = config.beforeAny; if (setup) sequence = sequence.concat(setup); const input = config.input; @@ -216,6 +207,57 @@ async function qualifyFiles({ config }) { for (let source of sequence) { log(config, "debug", `source: ${source}`); + + // Check if source is a heretto: reference + if (source.startsWith("heretto:")) { + const herettoName = source.substring(8); // Remove "heretto:" prefix + const herettoConfig = config?.integrations?.heretto?.find( + (h) => h.name === herettoName + ); + + if (!herettoConfig) { + log( + config, + "warning", + `Heretto integration "${herettoName}" not found in config. Skipping.` + ); + continue; + } + + // Load Heretto content if not already loaded + if (!herettoConfig.outputPath) { + try { + const outputPath = await loadHerettoContent(herettoConfig, log, config); + if (outputPath) { + herettoConfig.outputPath = outputPath; + log(config, "debug", `Adding Heretto output path: ${outputPath}`); + // Insert the output path into the sequence for processing + const currentIndex = sequence.indexOf(source); + sequence.splice(currentIndex + 1, 0, outputPath); + } else { + log( + config, + "warning", + `Failed to load Heretto content for "${herettoName}". Skipping.` + ); + } + } catch (error) { + log( + config, + "warning", + `Failed to load Heretto content from "${herettoName}": ${error.message}` + ); + } + } else { + // Already loaded, add to sequence if not already there + if (!sequence.includes(herettoConfig.outputPath)) { + const currentIndex = sequence.indexOf(source); + sequence.splice(currentIndex + 1, 0, herettoConfig.outputPath); + } + } + continue; + } + // Check if source is a URL let isURL = source.startsWith("http://") || source.startsWith("https://"); // If URL, fetch file and place in temp directory From 300c1b1d74ea4b0e3090de00f6a6406ec24229f1 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 5 Dec 2025 00:57:02 -0800 Subject: [PATCH 06/17] Ignore already processed ditamaps --- src/utils.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index e8a5e89..c166873 100644 --- a/src/utils.js +++ b/src/utils.js @@ -205,6 +205,8 @@ async function qualifyFiles({ config }) { return []; } + const ignoredDitaMaps = []; + for (let source of sequence) { log(config, "debug", `source: ${source}`); @@ -234,6 +236,7 @@ async function qualifyFiles({ config }) { // Insert the output path into the sequence for processing const currentIndex = sequence.indexOf(source); sequence.splice(currentIndex + 1, 0, outputPath); + ignoredDitaMaps.push(outputPath); // DITA maps are already processed in Heretto } else { log( config, @@ -277,13 +280,15 @@ async function qualifyFiles({ config }) { if ( isFile && path.extname(source) === ".ditamap" && - config.processDitaMap + !ignoredDitaMaps.some((ignored) => source.includes(ignored)) && + config.processDitaMaps ) { const ditaOutput = await processDitaMap({ config, source }); if (ditaOutput) { // Add output directory to to sequence right after the ditamap file const currentIndex = sequence.indexOf(source); sequence.splice(currentIndex + 1, 0, ditaOutput); + ignoredDitaMaps.push(ditaOutput); // DITA maps are already processed locally } continue; } From ad062a710db4d487ab3c95b754d17d487c51a214 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 5 Dec 2025 01:00:44 -0800 Subject: [PATCH 07/17] Revise heretto tests --- src/heretto.test.js | 153 +++++++++++++++++++++++++++++++++----------- 1 file changed, 114 insertions(+), 39 deletions(-) diff --git a/src/heretto.test.js b/src/heretto.test.js index e646511..af3f17e 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -12,7 +12,6 @@ before(async function () { describe("Heretto Integration", function () { let heretto; - let axiosStub; let axiosCreateStub; let mockClient; @@ -81,39 +80,43 @@ describe("Heretto Integration", function () { mockLog.reset(); }); - it("should return existing scenario if found", async function () { + it("should return scenarioId and fileId when valid scenario is found", async function () { const existingScenario = { id: "scenario-123", name: "Doc Detective", }; - mockClient.get.resolves({ - data: { content: [existingScenario, { id: "other", name: "Other" }] }, - }); + const scenarioParameters = { + content: [ + { name: "transtype", options: [{ value: "dita" }] }, + { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, + { type: "file_uuid_picker", value: "file-uuid-456" }, + ], + }; + + mockClient.get + .onFirstCall().resolves({ + data: { content: [existingScenario, { id: "other", name: "Other" }] }, + }) + .onSecondCall().resolves({ data: scenarioParameters }); const result = await heretto.findScenario(mockClient, mockLog, mockConfig); - expect(result).to.deep.equal(existingScenario); - expect(mockClient.get.calledOnce).to.be.true; + expect(result).to.deep.equal({ scenarioId: "scenario-123", fileId: "file-uuid-456" }); + expect(mockClient.get.calledTwice).to.be.true; expect(mockClient.post.called).to.be.false; }); - it("should create scenario if not found", async function () { - const newScenario = { - id: "new-scenario-123", - name: "Doc Detective", - }; - + it("should return null if scenario is not found", async function () { mockClient.get.resolves({ data: { content: [{ id: "other", name: "Other Scenario" }] }, }); - mockClient.post.resolves({ data: newScenario }); const result = await heretto.findScenario(mockClient, mockLog, mockConfig); - expect(result).to.deep.equal(newScenario); + expect(result).to.be.null; expect(mockClient.get.calledOnce).to.be.true; - expect(mockClient.post.calledOnce).to.be.true; + expect(mockClient.post.called).to.be.false; }); it("should return null if scenario fetch fails", async function () { @@ -124,9 +127,74 @@ describe("Heretto Integration", function () { expect(result).to.be.null; }); - it("should return null if scenario creation fails", async function () { - mockClient.get.resolves({ data: { content: [] } }); - mockClient.post.rejects(new Error("Permission denied")); + it("should return null if transtype parameter is incorrect", async function () { + const existingScenario = { + id: "scenario-123", + name: "Doc Detective", + }; + + const scenarioParameters = { + content: [ + { name: "transtype", options: [{ value: "html5" }] }, + { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, + { type: "file_uuid_picker", value: "file-uuid-456" }, + ], + }; + + mockClient.get + .onFirstCall().resolves({ + data: { content: [existingScenario] }, + }) + .onSecondCall().resolves({ data: scenarioParameters }); + + const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + + expect(result).to.be.null; + }); + + it("should return null if tool-kit-name parameter is incorrect", async function () { + const existingScenario = { + id: "scenario-123", + name: "Doc Detective", + }; + + const scenarioParameters = { + content: [ + { name: "transtype", options: [{ value: "dita" }] }, + { name: "tool-kit-name", value: "default/dita-ot-4.0.0" }, + { type: "file_uuid_picker", value: "file-uuid-456" }, + ], + }; + + mockClient.get + .onFirstCall().resolves({ + data: { content: [existingScenario] }, + }) + .onSecondCall().resolves({ data: scenarioParameters }); + + const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + + expect(result).to.be.null; + }); + + it("should return null if file_uuid_picker parameter is missing", async function () { + const existingScenario = { + id: "scenario-123", + name: "Doc Detective", + }; + + const scenarioParameters = { + content: [ + { name: "transtype", options: [{ value: "dita" }] }, + { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, + ], + }; + + mockClient.get + .onFirstCall().resolves({ + data: { content: [existingScenario] }, + }) + .onSecondCall().resolves({ data: scenarioParameters }); const result = await heretto.findScenario(mockClient, mockLog, mockConfig); @@ -137,7 +205,7 @@ describe("Heretto Integration", function () { describe("triggerPublishingJob", function () { it("should trigger a publishing job", async function () { const expectedJob = { - id: "job-123", + jobId: "job-123", status: "PENDING", }; @@ -147,8 +215,8 @@ describe("Heretto Integration", function () { expect(result).to.deep.equal(expectedJob); expect(mockClient.post.calledOnce).to.be.true; - expect(mockClient.post.firstCall.args[0]).to.equal("/files/file-uuid/publishing-jobs"); - expect(mockClient.post.firstCall.args[1]).to.deep.equal({ scenarioId: "scenario-id" }); + expect(mockClient.post.firstCall.args[0]).to.equal("/files/file-uuid/publishes"); + expect(mockClient.post.firstCall.args[1]).to.deep.equal({ scenario: "scenario-id", parameters: [] }); }); }); @@ -160,10 +228,10 @@ describe("Heretto Integration", function () { mockLog.reset(); }); - it("should return completed job", async function () { + it("should return completed job when status.result is SUCCESS", async function () { const completedJob = { id: "job-123", - status: "COMPLETED", + status: { status: "COMPLETED", result: "SUCCESS" }, }; mockClient.get.resolves({ data: completedJob }); @@ -173,11 +241,10 @@ describe("Heretto Integration", function () { expect(result).to.deep.equal(completedJob); }); - it("should return null for failed job", async function () { + it("should return null when status.result is FAIL", async function () { const failedJob = { id: "job-123", - status: "FAILED", - errorMessage: "Build failed", + status: { status: "FAILED", result: "FAIL" }, }; mockClient.get.resolves({ data: failedJob }); @@ -192,9 +259,9 @@ describe("Heretto Integration", function () { const clock = sinon.useFakeTimers(); mockClient.get - .onFirstCall().resolves({ data: { id: "job-123", status: "PENDING" } }) - .onSecondCall().resolves({ data: { id: "job-123", status: "PROCESSING" } }) - .onThirdCall().resolves({ data: { id: "job-123", status: "COMPLETED" } }); + .onFirstCall().resolves({ data: { id: "job-123", status: { status: "PENDING", result: null } } }) + .onSecondCall().resolves({ data: { id: "job-123", status: { status: "PROCESSING", result: null } } }) + .onThirdCall().resolves({ data: { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" } } }); const pollPromise = heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); @@ -205,7 +272,7 @@ describe("Heretto Integration", function () { const result = await pollPromise; - expect(result.status).to.equal("COMPLETED"); + expect(result.status.result).to.equal("SUCCESS"); expect(mockClient.get.callCount).to.equal(3); clock.restore(); @@ -220,13 +287,12 @@ describe("Heretto Integration", function () { mockLog.reset(); }); - it("should return null if scenario creation fails", async function () { + it("should return null if scenario lookup fails", async function () { const herettoConfig = { name: "test-heretto", organizationId: "test-org", username: "user@example.com", apiToken: "token123", - fileId: "file-uuid", }; // Scenario fetch fails @@ -237,19 +303,28 @@ describe("Heretto Integration", function () { expect(result).to.be.null; }); - it("should return null if publishing job fails", async function () { + it("should return null if publishing job creation fails", async function () { const herettoConfig = { name: "test-heretto", organizationId: "test-org", username: "user@example.com", apiToken: "token123", - fileId: "file-uuid", }; - // Scenario exists - mockClient.get.onFirstCall().resolves({ - data: { content: [{ id: "scenario-123", name: "Doc Detective" }] }, - }); + const scenarioParameters = { + content: [ + { name: "transtype", options: [{ value: "dita" }] }, + { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, + { type: "file_uuid_picker", value: "file-uuid-456" }, + ], + }; + + // Scenario exists with valid parameters + mockClient.get + .onFirstCall().resolves({ + data: { content: [{ id: "scenario-123", name: "Doc Detective" }] }, + }) + .onSecondCall().resolves({ data: scenarioParameters }); // Job creation fails mockClient.post.rejects(new Error("Job creation failed")); From b3407a6521c4eb5227280a3fb3c0066674d6d616 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 5 Dec 2025 01:23:24 -0800 Subject: [PATCH 08/17] Add .env to .gitignore and update Heretto integration schemas with example configuration --- package.json | 2 +- src/heretto.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index f0fa383..c295638 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "doc-detective-common": "file:../common", "dotenv": "^17.2.3", "json-schema-faker": "^0.5.9", - "posthog-node": "^5.15.0" + "posthog-node": "^5.17.2" }, "devDependencies": { "body-parser": "^2.2.1", diff --git a/src/heretto.js b/src/heretto.js index ef29e3f..b2a6939 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -43,6 +43,7 @@ function createApiClient(herettoConfig) { ); return axios.create({ baseURL: getBaseUrl(herettoConfig.organizationId), + timeout: 30000, // 30 second timeout for individual requests headers: { Authorization: `Basic ${authHeader}`, "Content-Type": "application/json", @@ -131,7 +132,7 @@ async function findScenario(client, log, config) { return null; } - // Make sure that senarioParameters.content has an object with name="tool-kit-name" and value="default/dita-ot-3.6.1" + // Make sure that scenarioParameters.content has an object with name="tool-kit-name" and value="default/dita-ot-3.6.1" const toolKitParam = scenarioParameters.content.find( (param) => param.name === "tool-kit-name" ); @@ -413,7 +414,7 @@ async function loadHerettoContent(herettoConfig, log, config) { log( config, "warning", - `Failed to load Heretto "${herettoConfig.name}": ${error}` + `Failed to load Heretto "${herettoConfig.name}": ${error.message}` ); return null; } From ae4d4c42fdb30ac8d5c3dd877084239a959ec2a7 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 5 Dec 2025 01:30:54 -0800 Subject: [PATCH 09/17] Longer timeout for downloads --- src/heretto.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/heretto.js b/src/heretto.js index b2a6939..fb6eb5d 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -298,6 +298,7 @@ async function downloadAndExtractOutput( `/files/${fileId}/publishes/${jobId}/assets-all`, { responseType: "arraybuffer", + timeout: 300000, // 5 minutes for downloads headers: { Accept: "application/octet-stream", }, From e6a3c660d24f06ca0152e41cbd1a68181ea8c490 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Mon, 8 Dec 2025 09:04:38 -0800 Subject: [PATCH 10/17] Loosen transtype validation --- src/heretto.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/heretto.js b/src/heretto.js index fb6eb5d..50a7ee1 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -123,7 +123,7 @@ async function findScenario(client, log, config) { const transtypeParam = scenarioParameters.content.find( (param) => param.name === "transtype" ); - if (!transtypeParam || !transtypeParam.options || transtypeParam.options[0].value !== "dita") { + if (!transtypeParam || transtypeParam.value !== "dita") { log( config, "error", @@ -136,11 +136,11 @@ async function findScenario(client, log, config) { const toolKitParam = scenarioParameters.content.find( (param) => param.name === "tool-kit-name" ); - if (!toolKitParam || !toolKitParam.value || toolKitParam.value !== "default/dita-ot-3.6.1") { + if (!toolKitParam || !toolKitParam.value) { log( config, "error", - `Existing "${SCENARIO_NAME}" scenario has incorrect "tool-kit-name" parameter settings. Make sure it is set to "default/dita-ot-3.6.1".` + `Existing "${SCENARIO_NAME}" scenario has incorrect "tool-kit-name" parameter settings".` ); return null; } From 8f93e2476c0803789720195399163a35fe23873a Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 12 Dec 2025 21:07:13 -0800 Subject: [PATCH 11/17] Use job.jobId when downloading Heretto output --- src/heretto.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heretto.js b/src/heretto.js index 50a7ee1..1337a5d 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -404,7 +404,7 @@ async function loadHerettoContent(herettoConfig, log, config) { const outputPath = await downloadAndExtractOutput( client, scenario.fileId, - completedJob.id, + job.jobId, herettoConfig.name, log, config From 84e3e1c03b1ee11e0f23c5ff37f1f2bbdf9768fd Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 12 Dec 2025 21:25:33 -0800 Subject: [PATCH 12/17] Added senarioName to heretto integration --- package-lock.json | 17 +++++++++-------- src/heretto.js | 39 ++++++++++----------------------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6a28780..e77d7c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "doc-detective-common": "file:../common", "dotenv": "^17.2.3", "json-schema-faker": "^0.5.9", - "posthog-node": "^5.15.0" + "posthog-node": "^5.17.2" }, "devDependencies": { "body-parser": "^2.2.1", @@ -144,9 +144,9 @@ } }, "node_modules/@posthog/core": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.6.0.tgz", - "integrity": "sha512-Tbh8UACwbb7jFdDC7wwXHtfNzO+4wKh3VbyMHmp2UBe6w1jliJixexTJNfkqdGZm+ht3M10mcKvGGPnoZ2zLBg==", + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.7.1.tgz", + "integrity": "sha512-kjK0eFMIpKo9GXIbts8VtAknsoZ18oZorANdtuTj1CbgS28t4ZVq//HAWhnxEuXRTrtkd+SUJ6Ux3j2Af8NCuA==", "license": "MIT", "dependencies": { "cross-spawn": "^7.0.6" @@ -1352,6 +1352,7 @@ "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.4.0.tgz", "integrity": "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==", "license": "MIT", + "peer": true, "engines": { "node": ">= 10.16.0" } @@ -1769,12 +1770,12 @@ "license": "ISC" }, "node_modules/posthog-node": { - "version": "5.15.0", - "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.15.0.tgz", - "integrity": "sha512-Q70DGmUoy/Lavrq+qn2shIj/EQSeyaz6huIZQlLE6y8Xa6xtZw+Jg1lyr2nC1sEEFPCU1X7zkNvP6mjbYwqjYA==", + "version": "5.17.2", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.17.2.tgz", + "integrity": "sha512-lz3YJOr0Nmiz0yHASaINEDHqoV+0bC3eD8aZAG+Ky292dAnVYul+ga/dMX8KCBXg8hHfKdxw0SztYD5j6dgUqQ==", "license": "MIT", "dependencies": { - "@posthog/core": "1.6.0" + "@posthog/core": "1.7.1" }, "engines": { "node": ">=20" diff --git a/src/heretto.js b/src/heretto.js index 1337a5d..c7dd0c6 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -8,7 +8,7 @@ const AdmZip = require("adm-zip"); // Internal constants - not exposed to users const POLLING_INTERVAL_MS = 5000; const POLLING_TIMEOUT_MS = 300000; // 5 minutes -const SCENARIO_NAME = "Doc Detective"; +const DEFAULT_SCENARIO_NAME = "Doc Detective"; const SCENARIO_DESCRIPTION = "Normalized DITA output for Doc Detective testing"; /** @@ -68,25 +68,6 @@ async function getPublishingScenarioParameters(client, scenarioId) { return response.data; } -/** - * Creates a new publishing scenario in Heretto. - * @param {Object} client - Configured axios instance - * @returns {Promise} Created scenario object - */ -async function createPublishingScenario(client) { - const scenarioConfig = { - name: SCENARIO_NAME, - description: SCENARIO_DESCRIPTION, - outputFormat: "dita", - engine: "dita-ot", - parameters: { - transtype: "dita", - }, - }; - const response = await client.post("/publishing-scenarios", scenarioConfig); - return response.data; -} - // TODO: Remove scenario creation. Users have to create it manually with specific settings. /** * Finds an existing "Doc Detective" scenario or creates one if missing. @@ -95,13 +76,13 @@ async function createPublishingScenario(client) { * @param {Object} config - Doc Detective config for logging * @returns {Promise} Scenario object or null if creation failed */ -async function findScenario(client, log, config) { +async function findScenario(client, log, config, scenarioName) { try { const scenarios = await getPublishingScenarios(client); - const foundScenario = scenarios.find((s) => s.name === SCENARIO_NAME); + const foundScenario = scenarios.find((s) => s.name === scenarioName); if (!foundScenario) { - log(config, "error", `No existing "${SCENARIO_NAME}" scenario found.`); + log(config, "error", `No existing "${scenarioName}" scenario found.`); return null; } @@ -127,7 +108,7 @@ async function findScenario(client, log, config) { log( config, "error", - `Existing "${SCENARIO_NAME}" scenario has incorrect "transtype" parameter settings. Make sure it is set to "dita".` + `Existing "${DEFAULT_SCENARIO_NAME}" scenario has incorrect "transtype" parameter settings. Make sure it is set to "dita".` ); return null; } @@ -140,7 +121,7 @@ async function findScenario(client, log, config) { log( config, "error", - `Existing "${SCENARIO_NAME}" scenario has incorrect "tool-kit-name" parameter settings".` + `Existing "${DEFAULT_SCENARIO_NAME}" scenario has incorrect "tool-kit-name" parameter settings".` ); return null; } @@ -153,7 +134,7 @@ async function findScenario(client, log, config) { log( config, "error", - `Existing "${SCENARIO_NAME}" scenario has incorrect "file_uuid_picker" parameter settings. Make sure it has a valid value.` + `Existing "${DEFAULT_SCENARIO_NAME}" scenario has incorrect "file_uuid_picker" parameter settings. Make sure it has a valid value.` ); return null; } @@ -161,7 +142,7 @@ async function findScenario(client, log, config) { log( config, "debug", - `Found existing "${SCENARIO_NAME}" scenario: ${foundScenario.id}` + `Found existing "${DEFAULT_SCENARIO_NAME}" scenario: ${foundScenario.id}` ); return { scenarioId: foundScenario.id, fileId: fileUuidPickerParam.value }; } catch (error) { @@ -352,7 +333,7 @@ async function loadHerettoContent(herettoConfig, log, config) { const client = createApiClient(herettoConfig); // Find or create the Doc Detective publishing scenario - const scenario = await findScenario(client, log, config); + const scenario = await findScenario(client, log, config, herettoConfig.scenarioName); if (!scenario) { log( config, @@ -432,5 +413,5 @@ module.exports = { // Export constants for testing POLLING_INTERVAL_MS, POLLING_TIMEOUT_MS, - SCENARIO_NAME, + DEFAULT_SCENARIO_NAME, }; From b5c31b78836c875456bbfd5e4dc25e78a41da974 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 16 Dec 2025 11:40:58 -0800 Subject: [PATCH 13/17] Set dep version --- package-lock.json | 76 +++++++++++++++++++++++++++++++++-------------- package.json | 2 +- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index e77d7c9..3571fc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "file:../common", + "doc-detective-common": "3.6.0-dev.1", "dotenv": "^17.2.3", "json-schema-faker": "^0.5.9", "posthog-node": "^5.17.2" @@ -29,25 +29,6 @@ "yaml": "^2.8.2" } }, - "../common": { - "name": "doc-detective-common", - "version": "3.6.0", - "license": "AGPL-3.0-only", - "dependencies": { - "@apidevtools/json-schema-ref-parser": "^15.1.3", - "ajv": "^8.17.1", - "ajv-errors": "^3.0.0", - "ajv-formats": "^3.0.1", - "ajv-keywords": "^5.1.0", - "axios": "^1.13.2", - "yaml": "^2.8.2" - }, - "devDependencies": { - "chai": "^6.2.1", - "mocha": "^11.7.5", - "sinon": "^21.0.0" - } - }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "15.1.3", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-15.1.3.tgz", @@ -251,6 +232,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -262,6 +244,44 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-errors": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-3.0.0.tgz", + "integrity": "sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^8.0.1" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -693,8 +713,19 @@ } }, "node_modules/doc-detective-common": { - "resolved": "../common", - "link": true + "version": "3.6.0-dev.1", + "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.6.0-dev.1.tgz", + "integrity": "sha512-e+3FNyqjhPUZRq+4A1t7G+au07RZockzCHdQ6LDaQQySPtAiNSO42v48ylbHIu4ZOn06SO933rVJe/b+e1GVdw==", + "license": "AGPL-3.0-only", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^15.1.3", + "ajv": "^8.17.1", + "ajv-errors": "^3.0.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0", + "axios": "^1.13.2", + "yaml": "^2.8.2" + } }, "node_modules/dotenv": { "version": "17.2.3", @@ -2557,7 +2588,6 @@ "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", - "dev": true, "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/package.json b/package.json index c295638..e4cc2ed 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "file:../common", + "doc-detective-common": "3.6.0-dev.1", "dotenv": "^17.2.3", "json-schema-faker": "^0.5.9", "posthog-node": "^5.17.2" From b0cedc525546d15323c0739745e12e10002dc5e7 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 16 Dec 2025 13:58:52 -0800 Subject: [PATCH 14/17] Update findScenario function and constants in Heretto integration tests - Change transtype options structure to a single value - Pass scenario name to findScenario function calls - Update constant name from SCENARIO_NAME to DEFAULT_SCENARIO_NAME --- src/heretto.test.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/heretto.test.js b/src/heretto.test.js index af3f17e..ba202b3 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -88,7 +88,7 @@ describe("Heretto Integration", function () { const scenarioParameters = { content: [ - { name: "transtype", options: [{ value: "dita" }] }, + { name: "transtype", value: "dita" }, { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, { type: "file_uuid_picker", value: "file-uuid-456" }, ], @@ -100,7 +100,7 @@ describe("Heretto Integration", function () { }) .onSecondCall().resolves({ data: scenarioParameters }); - const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); expect(result).to.deep.equal({ scenarioId: "scenario-123", fileId: "file-uuid-456" }); expect(mockClient.get.calledTwice).to.be.true; @@ -112,7 +112,7 @@ describe("Heretto Integration", function () { data: { content: [{ id: "other", name: "Other Scenario" }] }, }); - const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); expect(result).to.be.null; expect(mockClient.get.calledOnce).to.be.true; @@ -122,7 +122,7 @@ describe("Heretto Integration", function () { it("should return null if scenario fetch fails", async function () { mockClient.get.rejects(new Error("Network error")); - const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); expect(result).to.be.null; }); @@ -147,7 +147,7 @@ describe("Heretto Integration", function () { }) .onSecondCall().resolves({ data: scenarioParameters }); - const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); expect(result).to.be.null; }); @@ -172,7 +172,7 @@ describe("Heretto Integration", function () { }) .onSecondCall().resolves({ data: scenarioParameters }); - const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); expect(result).to.be.null; }); @@ -196,7 +196,7 @@ describe("Heretto Integration", function () { }) .onSecondCall().resolves({ data: scenarioParameters }); - const result = await heretto.findScenario(mockClient, mockLog, mockConfig); + const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); expect(result).to.be.null; }); @@ -339,7 +339,7 @@ describe("Heretto Integration", function () { it("should export expected constants", function () { expect(heretto.POLLING_INTERVAL_MS).to.equal(5000); expect(heretto.POLLING_TIMEOUT_MS).to.equal(300000); - expect(heretto.SCENARIO_NAME).to.equal("Doc Detective"); + expect(heretto.DEFAULT_SCENARIO_NAME).to.equal("Doc Detective"); }); }); }); From 877ed1bb293beb6fa2ab1873c489df3377b491e3 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 16 Dec 2025 18:17:20 -0800 Subject: [PATCH 15/17] Refactor scenario parameters in Heretto integration tests - Change 'options' to 'value' for 'transtype' in multiple test cases - Update test description to reflect that 'tool-kit-name' parameter can be missing - Add 'scenarioName' to the context in relevant tests --- src/heretto.test.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/heretto.test.js b/src/heretto.test.js index ba202b3..574b0aa 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -135,7 +135,7 @@ describe("Heretto Integration", function () { const scenarioParameters = { content: [ - { name: "transtype", options: [{ value: "html5" }] }, + { name: "transtype", value: "html5" }, { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, { type: "file_uuid_picker", value: "file-uuid-456" }, ], @@ -152,7 +152,7 @@ describe("Heretto Integration", function () { expect(result).to.be.null; }); - it("should return null if tool-kit-name parameter is incorrect", async function () { + it("should return null if tool-kit-name parameter is missing", async function () { const existingScenario = { id: "scenario-123", name: "Doc Detective", @@ -160,8 +160,7 @@ describe("Heretto Integration", function () { const scenarioParameters = { content: [ - { name: "transtype", options: [{ value: "dita" }] }, - { name: "tool-kit-name", value: "default/dita-ot-4.0.0" }, + { name: "transtype", value: "dita" }, { type: "file_uuid_picker", value: "file-uuid-456" }, ], }; @@ -185,7 +184,7 @@ describe("Heretto Integration", function () { const scenarioParameters = { content: [ - { name: "transtype", options: [{ value: "dita" }] }, + { name: "transtype", value: "dita" }, { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, ], }; @@ -293,6 +292,7 @@ describe("Heretto Integration", function () { organizationId: "test-org", username: "user@example.com", apiToken: "token123", + scenarioName: "Doc Detective", }; // Scenario fetch fails @@ -309,11 +309,12 @@ describe("Heretto Integration", function () { organizationId: "test-org", username: "user@example.com", apiToken: "token123", + scenarioName: "Doc Detective", }; const scenarioParameters = { content: [ - { name: "transtype", options: [{ value: "dita" }] }, + { name: "transtype", value: "dita" }, { name: "tool-kit-name", value: "default/dita-ot-3.6.1" }, { type: "file_uuid_picker", value: "file-uuid-456" }, ], From c5d03d68d0b756e10a4284bf2abed5451b036bdf Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 16 Dec 2025 18:37:21 -0800 Subject: [PATCH 16/17] Refactor Heretto integration code - Update timeout constants for API requests and downloads - Modify findScenario function to use scenarioName parameter - Remove unused scenario description constant - Simplify temp directory creation logic in download function - Clean up test file by removing unnecessary imports --- src/heretto.js | 47 +++++++++++++++++++-------------------------- src/heretto.test.js | 4 ---- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/heretto.js b/src/heretto.js index c7dd0c6..b80e684 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -8,8 +8,9 @@ const AdmZip = require("adm-zip"); // Internal constants - not exposed to users const POLLING_INTERVAL_MS = 5000; const POLLING_TIMEOUT_MS = 300000; // 5 minutes +const API_REQUEST_TIMEOUT_MS = 30000; // 30 seconds for individual API requests +const DOWNLOAD_TIMEOUT_MS = 300000; // 5 minutes for downloads const DEFAULT_SCENARIO_NAME = "Doc Detective"; -const SCENARIO_DESCRIPTION = "Normalized DITA output for Doc Detective testing"; /** * Creates a Base64-encoded Basic Auth header from username and API token. @@ -43,7 +44,7 @@ function createApiClient(herettoConfig) { ); return axios.create({ baseURL: getBaseUrl(herettoConfig.organizationId), - timeout: 30000, // 30 second timeout for individual requests + timeout: API_REQUEST_TIMEOUT_MS, headers: { Authorization: `Basic ${authHeader}`, "Content-Type": "application/json", @@ -61,6 +62,12 @@ async function getPublishingScenarios(client) { return response.data.content || []; } +/** + * Fetches parameters for a specific publishing scenario. + * @param {Object} client - Configured axios instance + * @param {string} scenarioId - ID of the publishing scenario + * @returns {Promise} Scenario parameters object + */ async function getPublishingScenarioParameters(client, scenarioId) { const response = await client.get( `/publishes/scenarios/${scenarioId}/parameters` @@ -68,9 +75,8 @@ async function getPublishingScenarioParameters(client, scenarioId) { return response.data; } -// TODO: Remove scenario creation. Users have to create it manually with specific settings. /** - * Finds an existing "Doc Detective" scenario or creates one if missing. + * Finds an existing publishing scenario by name and validates its configuration. * @param {Object} client - Configured axios instance * @param {Function} log - Logging function * @param {Object} config - Doc Detective config for logging @@ -108,7 +114,7 @@ async function findScenario(client, log, config, scenarioName) { log( config, "error", - `Existing "${DEFAULT_SCENARIO_NAME}" scenario has incorrect "transtype" parameter settings. Make sure it is set to "dita".` + `Existing "${scenarioName}" scenario has incorrect "transtype" parameter settings. Make sure it is set to "dita".` ); return null; } @@ -121,7 +127,7 @@ async function findScenario(client, log, config, scenarioName) { log( config, "error", - `Existing "${DEFAULT_SCENARIO_NAME}" scenario has incorrect "tool-kit-name" parameter settings".` + `Existing "${scenarioName}" scenario has incorrect "tool-kit-name" parameter settings.` ); return null; } @@ -134,7 +140,7 @@ async function findScenario(client, log, config, scenarioName) { log( config, "error", - `Existing "${DEFAULT_SCENARIO_NAME}" scenario has incorrect "file_uuid_picker" parameter settings. Make sure it has a valid value.` + `Existing "${scenarioName}" scenario has incorrect "file_uuid_picker" parameter settings. Make sure it has a valid value.` ); return null; } @@ -142,7 +148,7 @@ async function findScenario(client, log, config, scenarioName) { log( config, "debug", - `Found existing "${DEFAULT_SCENARIO_NAME}" scenario: ${foundScenario.id}` + `Found existing "${scenarioName}" scenario: ${foundScenario.id}` ); return { scenarioId: foundScenario.id, fileId: fileUuidPickerParam.value }; } catch (error) { @@ -184,11 +190,6 @@ async function getJobStatus(client, fileId, jobId) { return response.data; } -async function getPublishingJobHistory(client, fileId) { - const response = await client.get(`/files/${fileId}/publishes`); - return response.data.content || []; -} - /** * Polls a publishing job until completion or timeout. * @param {Object} client - Configured axios instance @@ -206,7 +207,7 @@ async function pollJobStatus(client, fileId, jobId, log, config) { const job = await getJobStatus(client, fileId, jobId); log(config, "debug", `Job ${jobId} status: ${job?.status?.status}`); - if (job?.status?.result === "SUCCESS" ) { + if (job?.status?.result === "SUCCESS") { return job; } @@ -258,9 +259,7 @@ async function downloadAndExtractOutput( try { // Create temp directory if it doesn't exist const tempDir = `${os.tmpdir()}/doc-detective`; - if (!fs.existsSync(tempDir)) { - fs.mkdirSync(tempDir, { recursive: true }); - } + fs.mkdirSync(tempDir, { recursive: true }); // Create unique output directory based on heretto name and job ID const hash = crypto @@ -279,7 +278,7 @@ async function downloadAndExtractOutput( `/files/${fileId}/publishes/${jobId}/assets-all`, { responseType: "arraybuffer", - timeout: 300000, // 5 minutes for downloads + timeout: DOWNLOAD_TIMEOUT_MS, headers: { Accept: "application/octet-stream", }, @@ -332,8 +331,9 @@ async function loadHerettoContent(herettoConfig, log, config) { try { const client = createApiClient(herettoConfig); - // Find or create the Doc Detective publishing scenario - const scenario = await findScenario(client, log, config, herettoConfig.scenarioName); + // Find the Doc Detective publishing scenario + const scenarioName = herettoConfig.scenarioName || DEFAULT_SCENARIO_NAME; + const scenario = await findScenario(client, log, config, scenarioName); if (!scenario) { log( config, @@ -343,13 +343,6 @@ async function loadHerettoContent(herettoConfig, log, config) { return null; } - // const history = await getPublishingJobHistory(client, scenario.fileId); - // log( - // config, - // "debug", - // `Found ${history.length} previous publishing jobs for file ${scenario.fileId}` - // ); - // Trigger publishing job log( config, diff --git a/src/heretto.test.js b/src/heretto.test.js index 574b0aa..72535d2 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -1,9 +1,5 @@ -const assert = require("assert"); const sinon = require("sinon"); const proxyquire = require("proxyquire"); -const path = require("path"); -const fs = require("fs"); -const os = require("os"); before(async function () { const { expect } = await import("chai"); From 2da6d1546f8b275a0fd23b86c9d904c92de009ea Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 16 Dec 2025 22:21:48 -0800 Subject: [PATCH 17/17] Enhance Heretto integration with improved error handling and security - Update findScenario function to include scenarioName parameter and clarify return type - Implement path traversal protection in downloadAndExtractOutput function - Add tests for error handling in triggerPublishingJob and pollJobStatus functions - Create tests for downloadAndExtractOutput to validate ZIP extraction and malicious entry handling --- src/heretto.js | 26 +++++++- src/heretto.test.js | 141 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/src/heretto.js b/src/heretto.js index b80e684..c535e33 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -80,7 +80,8 @@ async function getPublishingScenarioParameters(client, scenarioId) { * @param {Object} client - Configured axios instance * @param {Function} log - Logging function * @param {Object} config - Doc Detective config for logging - * @returns {Promise} Scenario object or null if creation failed + * @param {string} scenarioName - Name of the scenario to find + * @returns {Promise} Object with scenarioId and fileId, or null if not found or invalid */ async function findScenario(client, log, config, scenarioName) { try { @@ -289,10 +290,29 @@ async function downloadAndExtractOutput( const zipPath = path.join(tempDir, `heretto_${hash}.zip`); fs.writeFileSync(zipPath, response.data); - // Extract ZIP contents + // Extract ZIP contents with path traversal protection log(config, "debug", `Extracting output to ${outputDir}...`); const zip = new AdmZip(zipPath); - zip.extractAllTo(outputDir, true); + const resolvedOutputDir = path.resolve(outputDir); + + // Validate and extract entries safely to prevent zip slip attacks + for (const entry of zip.getEntries()) { + const entryPath = path.join(outputDir, entry.entryName); + const resolvedPath = path.resolve(entryPath); + + // Ensure the resolved path is within outputDir + if (!resolvedPath.startsWith(resolvedOutputDir + path.sep) && resolvedPath !== resolvedOutputDir) { + log(config, "warning", `Skipping potentially malicious ZIP entry: ${entry.entryName}`); + continue; + } + + if (entry.isDirectory) { + fs.mkdirSync(resolvedPath, { recursive: true }); + } else { + fs.mkdirSync(path.dirname(resolvedPath), { recursive: true }); + fs.writeFileSync(resolvedPath, entry.getData()); + } + } // Clean up ZIP file fs.unlinkSync(zipPath); diff --git a/src/heretto.test.js b/src/heretto.test.js index 72535d2..c89ea82 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -1,5 +1,7 @@ const sinon = require("sinon"); const proxyquire = require("proxyquire"); +const path = require("path"); +const os = require("os"); before(async function () { const { expect } = await import("chai"); @@ -213,6 +215,17 @@ describe("Heretto Integration", function () { expect(mockClient.post.firstCall.args[0]).to.equal("/files/file-uuid/publishes"); expect(mockClient.post.firstCall.args[1]).to.deep.equal({ scenario: "scenario-id", parameters: [] }); }); + + it("should throw error when job creation fails", async function () { + mockClient.post.rejects(new Error("API error")); + + try { + await heretto.triggerPublishingJob(mockClient, "file-uuid", "scenario-id"); + expect.fail("Expected error to be thrown"); + } catch (error) { + expect(error.message).to.equal("API error"); + } + }); }); describe("pollJobStatus", function () { @@ -272,6 +285,34 @@ describe("Heretto Integration", function () { clock.restore(); }); + + it("should return null on timeout", async function () { + // Use fake timers to avoid waiting for real timeout + const clock = sinon.useFakeTimers(); + + // Always return PENDING status (never completes) + mockClient.get.resolves({ + data: { id: "job-123", status: { status: "PENDING", result: null } } + }); + + const pollPromise = heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + // Advance past the timeout + await clock.tickAsync(heretto.POLLING_TIMEOUT_MS + heretto.POLLING_INTERVAL_MS); + + const result = await pollPromise; + expect(result).to.be.null; + + clock.restore(); + }); + + it("should return null when polling error occurs", async function () { + mockClient.get.rejects(new Error("Network error")); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.be.null; + }); }); describe("loadHerettoContent", function () { @@ -332,6 +373,106 @@ describe("Heretto Integration", function () { }); }); + describe("downloadAndExtractOutput", function () { + let herettoWithMocks; + let fsMock; + let admZipMock; + let mockEntries; + const mockLog = sinon.stub(); + const mockConfig = { logLevel: "info" }; + + beforeEach(function () { + mockLog.reset(); + + // Mock ZIP entries + mockEntries = [ + { entryName: "file1.dita", isDirectory: false, getData: () => Buffer.from("content1") }, + { entryName: "subdir/", isDirectory: true, getData: () => Buffer.from("") }, + { entryName: "subdir/file2.dita", isDirectory: false, getData: () => Buffer.from("content2") }, + ]; + + // Mock AdmZip + admZipMock = sinon.stub().returns({ + getEntries: () => mockEntries, + extractAllTo: sinon.stub(), + }); + + // Mock fs + fsMock = { + mkdirSync: sinon.stub(), + writeFileSync: sinon.stub(), + unlinkSync: sinon.stub(), + }; + + // Create heretto with mocked dependencies + herettoWithMocks = proxyquire("../src/heretto", { + axios: { create: axiosCreateStub }, + fs: fsMock, + "adm-zip": admZipMock, + }); + }); + + it("should download and extract ZIP file successfully", async function () { + const zipContent = Buffer.from("mock zip content"); + mockClient.get.resolves({ data: zipContent }); + + const result = await herettoWithMocks.downloadAndExtractOutput( + mockClient, + "file-uuid", + "job-123", + "test-heretto", + mockLog, + mockConfig + ); + + expect(result).to.not.be.null; + expect(result).to.include("heretto_"); + expect(fsMock.mkdirSync.called).to.be.true; + expect(fsMock.writeFileSync.called).to.be.true; + expect(fsMock.unlinkSync.called).to.be.true; + }); + + it("should return null when download fails", async function () { + mockClient.get.rejects(new Error("Download failed")); + + const result = await herettoWithMocks.downloadAndExtractOutput( + mockClient, + "file-uuid", + "job-123", + "test-heretto", + mockLog, + mockConfig + ); + + expect(result).to.be.null; + }); + + it("should skip malicious ZIP entries with path traversal", async function () { + // Add malicious entry + mockEntries.push({ + entryName: "../../../etc/passwd", + isDirectory: false, + getData: () => Buffer.from("malicious") + }); + + const zipContent = Buffer.from("mock zip content"); + mockClient.get.resolves({ data: zipContent }); + + const result = await herettoWithMocks.downloadAndExtractOutput( + mockClient, + "file-uuid", + "job-123", + "test-heretto", + mockLog, + mockConfig + ); + + expect(result).to.not.be.null; + // The warning log should be called for the malicious entry + expect(mockLog.called).to.be.true; + }); + }); + describe("Constants", function () { it("should export expected constants", function () { expect(heretto.POLLING_INTERVAL_MS).to.equal(5000);