diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000..1293867 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,120 @@ +name: Integration Tests + +on: + push: + branches: + - main + - heretto + paths: + - 'src/heretto*.js' + - '.github/workflows/integration-tests.yml' + pull_request: + branches: + - main + paths: + - 'src/heretto*.js' + - '.github/workflows/integration-tests.yml' + workflow_dispatch: + # Allow manual triggering for testing + schedule: + # Run daily at 6:00 AM UTC to catch any API changes + - cron: '0 6 * * *' + +jobs: + heretto-integration-tests: + runs-on: ubuntu-latest + timeout-minutes: 15 + # Only run if secrets are available (not available on fork PRs) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run integration tests + env: + CI: 'true' + HERETTO_ORGANIZATION_ID: ${{ secrets.HERETTO_ORGANIZATION_ID }} + HERETTO_USERNAME: ${{ secrets.HERETTO_USERNAME }} + HERETTO_API_TOKEN: ${{ secrets.HERETTO_API_TOKEN }} + # HERETTO_SCENARIO_NAME is optional - defaults to 'Doc Detective' in the application + HERETTO_SCENARIO_NAME: ${{ secrets.HERETTO_SCENARIO_NAME }} + run: npm run test:integration + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-results + path: | + test-results/ + *.log + retention-days: 7 + + notify-on-failure: + runs-on: ubuntu-latest + needs: heretto-integration-tests + if: failure() && github.event_name == 'schedule' + steps: + - name: Create issue on failure + uses: actions/github-script@v7 + with: + script: | + const title = '🚨 Heretto Integration Tests Failed'; + const body = ` + ## Integration Test Failure + + The scheduled Heretto integration tests have failed. + + **Workflow Run:** [View Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + **Triggered:** ${{ github.event_name }} + **Branch:** ${{ github.ref_name }} + + Please investigate and fix the issue. + + ### Possible Causes + - Heretto API changes + - Expired or invalid API credentials + - Network connectivity issues + - Changes in test scenario configuration + + /cc @${{ github.repository_owner }} + `; + + // Check if an open issue already exists + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'integration-test-failure' + }); + + const existingIssue = issues.data.find(issue => issue.title === title); + + if (!existingIssue) { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['bug', 'integration-test-failure', 'automated'] + }); + } else { + // Add a comment to the existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existingIssue.number, + body: `Another failure detected on ${new Date().toISOString()}\n\n[Workflow Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})` + }); + } diff --git a/package-lock.json b/package-lock.json index 1b40437..ddd9c38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,19 +13,20 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "3.6.0-dev.1", + "doc-detective-common": "^3.6.0-dev.2", "dotenv": "^17.2.3", + "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", - "posthog-node": "^5.17.2" + "posthog-node": "^5.18.1" }, "devDependencies": { "body-parser": "^2.2.1", - "chai": "^6.2.1", + "chai": "^6.2.2", "express": "^5.2.1", "mocha": "^11.7.5", "proxyquire": "^2.1.3", "semver": "^7.7.3", - "sinon": "^21.0.0", + "sinon": "^21.0.1", "yaml": "^2.8.2" } }, @@ -125,9 +126,9 @@ } }, "node_modules/@posthog/core": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.7.1.tgz", - "integrity": "sha512-kjK0eFMIpKo9GXIbts8VtAknsoZ18oZorANdtuTj1CbgS28t4ZVq//HAWhnxEuXRTrtkd+SUJ6Ux3j2Af8NCuA==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.9.0.tgz", + "integrity": "sha512-j7KSWxJTUtNyKynLt/p0hfip/3I46dWU2dk+pt7dKRoz2l5CYueHuHK4EO7Wlgno5yo1HO4sc4s30MXMTICHJw==", "license": "MIT", "dependencies": { "cross-spawn": "^7.0.6" @@ -144,9 +145,9 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", - "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.1.0.tgz", + "integrity": "sha512-cqfapCxwTGsrR80FEgOoPsTonoefMBY7dnUEbQ+GRcved0jvkJLzvX6F4WtN+HBqbPX/SiFsIRUp+IrCW/2I2w==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -154,14 +155,13 @@ } }, "node_modules/@sinonjs/samsam": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.2.tgz", - "integrity": "sha512-v46t/fwnhejRSFTGqbpn9u+LQ9xJDse10gNnPgAcxgdoCDMXj/G2asWAC/8Qs+BAZDicX+MNZouXT1A7c83kVw==", + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.3.tgz", + "integrity": "sha512-hw6HbX+GyVZzmaYNh82Ecj1vdGZrqVIn/keDTg63IgAwiQPO+xCz99uG6Woqgb4tM0mUiFENKZ4cqd7IX94AXQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^3.0.1", - "lodash.get": "^4.4.2", "type-detect": "^4.1.0" } }, @@ -437,9 +437,9 @@ } }, "node_modules/chai": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.1.tgz", - "integrity": "sha512-p4Z49OGG5W/WBCPSS/dH3jQ73kD6tiMmUM+bckNK6Jr5JHMG3k9bg/BvKR8lKmtVBKmOiuVaV2ws8s9oSbwysg==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", "engines": { @@ -713,9 +713,9 @@ } }, "node_modules/doc-detective-common": { - "version": "3.6.0-dev.1", - "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.6.0-dev.1.tgz", - "integrity": "sha512-e+3FNyqjhPUZRq+4A1t7G+au07RZockzCHdQ6LDaQQySPtAiNSO42v48ylbHIu4ZOn06SO933rVJe/b+e1GVdw==", + "version": "3.6.0-dev.2", + "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.6.0-dev.2.tgz", + "integrity": "sha512-6wrfWvUSNVh8I1ccqCI7Z0ICxK5j59X2XttLsspcfiSTA42Rsm7jTlFrbMFxQOGBukyVa3mIi44maF9FCTkvdw==", "license": "AGPL-3.0-only", "dependencies": { "@apidevtools/json-schema-ref-parser": "^15.1.3", @@ -953,6 +953,24 @@ "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" }, + "node_modules/fast-xml-parser": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.3.tgz", + "integrity": "sha512-2O3dkPAAC6JavuMm8+4+pgTk+5hoAs+CjZ+sWcQLkX9+/tHRuTkQh/Oaifr8qDmZ8iEHb771Ea6G8CdwkrgvYA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fill-keys": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", @@ -1473,14 +1491,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", - "deprecated": "This package is deprecated. Use the optional chaining (?.) operator instead.", - "dev": true, - "license": "MIT" - }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -1801,12 +1811,12 @@ "license": "ISC" }, "node_modules/posthog-node": { - "version": "5.17.2", - "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.17.2.tgz", - "integrity": "sha512-lz3YJOr0Nmiz0yHASaINEDHqoV+0bC3eD8aZAG+Ky292dAnVYul+ga/dMX8KCBXg8hHfKdxw0SztYD5j6dgUqQ==", + "version": "5.18.1", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.18.1.tgz", + "integrity": "sha512-Hi7cRqAlvuEitdiurXJFdMip+BxcwYoX66at5RErMVP91V+Ph9BspGiawC3mJx/4znjwUjF29kAhf8oZQ2uJ5Q==", "license": "MIT", "dependencies": { - "@posthog/core": "1.7.1" + "@posthog/core": "1.9.0" }, "engines": { "node": ">=20" @@ -1844,9 +1854,9 @@ } }, "node_modules/qs": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", - "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -2185,16 +2195,16 @@ } }, "node_modules/sinon": { - "version": "21.0.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-21.0.0.tgz", - "integrity": "sha512-TOgRcwFPbfGtpqvZw+hyqJDvqfapr1qUlOizROIk4bBLjlsjlB00Pg6wMFXNtJRpu+eCZuVOaLatG7M8105kAw==", + "version": "21.0.1", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-21.0.1.tgz", + "integrity": "sha512-Z0NVCW45W8Mg5oC/27/+fCqIHFnW8kpkFOq0j9XJIev4Ld0mKmERaZv5DMLAb9fGCevjKwaEeIQz5+MBXfZcDw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^3.0.1", - "@sinonjs/fake-timers": "^13.0.5", - "@sinonjs/samsam": "^8.0.1", - "diff": "^7.0.0", + "@sinonjs/fake-timers": "^15.1.0", + "@sinonjs/samsam": "^8.0.3", + "diff": "^8.0.2", "supports-color": "^7.2.0" }, "funding": { @@ -2202,6 +2212,16 @@ "url": "https://opencollective.com/sinon" } }, + "node_modules/sinon/node_modules/diff": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.2.tgz", + "integrity": "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/sinon/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2339,6 +2359,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", + "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", diff --git a/package.json b/package.json index b6e7b26..94533b6 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,9 @@ "description": "Detect and resolve docs into Doc Detective tests.", "main": "src/index.js", "scripts": { - "test": "mocha src/*.test.js", + "test": "mocha src/*.test.js --ignore src/*.integration.test.js", + "test:integration": "mocha src/*.integration.test.js --timeout 600000", + "test:all": "mocha src/*.test.js --timeout 600000", "dev": "node dev" }, "repository": { @@ -28,19 +30,20 @@ "adm-zip": "^0.5.16", "ajv": "^8.17.1", "axios": "^1.13.2", - "doc-detective-common": "3.6.0-dev.1", + "doc-detective-common": "^3.6.0-dev.2", "dotenv": "^17.2.3", + "fast-xml-parser": "^5.3.3", "json-schema-faker": "^0.5.9", - "posthog-node": "^5.17.2" + "posthog-node": "^5.18.1" }, "devDependencies": { "body-parser": "^2.2.1", - "chai": "^6.2.1", + "chai": "^6.2.2", "express": "^5.2.1", "mocha": "^11.7.5", "proxyquire": "^2.1.3", "semver": "^7.7.3", - "sinon": "^21.0.0", + "sinon": "^21.0.1", "yaml": "^2.8.2" } } diff --git a/src/config.js b/src/config.js index 8bacb3a..0dc59f3 100644 --- a/src/config.js +++ b/src/config.js @@ -88,6 +88,7 @@ let defaultFileTypes = { step: [ "<\\?doc-detective\\s+step\\s+([\\s\\S]*?)\\s*\\?>", "", + '([\\s\\S]*?)<\\/data>', ], }, markup: [ @@ -124,7 +125,7 @@ let defaultFileTypes = { { name: "runShellCmdWithCodeblock", regex: [ - "(?:[Rr]un|[Ee]xecute)\\s+(?:the\\s+)?(?:following\\s+)?(?:command)[^<]*<\\/cmd>\\s*\\s*]*outputclass=\"(?:shell|bash)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + '(?:[Rr]un|[Ee]xecute)\\s+(?:the\\s+)?(?:following\\s+)?(?:command)[^<]*<\\/cmd>\\s*\\s*]*outputclass="(?:shell|bash)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -133,27 +134,21 @@ let defaultFileTypes = { }, }, ], - }, + }, // Inline Elements - for finding UI elements and text { name: "findUiControl", - regex: [ - "([^<]+)<\\/uicontrol>", - ], + regex: ["([^<]+)<\\/uicontrol>"], actions: ["find"], }, { name: "verifyWindowTitle", - regex: [ - "([^<]+)<\\/wintitle>", - ], + regex: ["([^<]+)<\\/wintitle>"], actions: ["find"], }, { name: "EnterKey", - regex: [ - "(?:[Pp]ress)\\s+Enter<\\/shortcut>", - ], + regex: ["(?:[Pp]ress)\\s+Enter<\\/shortcut>"], actions: [ { type: { @@ -164,9 +159,7 @@ let defaultFileTypes = { }, { name: "executeCmdName", - regex: [ - "(?:[Ee]xecute|[Rr]un)\\s+([^<]+)<\\/cmdname>", - ], + regex: ["(?:[Ee]xecute|[Rr]un)\\s+([^<]+)<\\/cmdname>"], actions: [ { runShell: { @@ -175,7 +168,7 @@ let defaultFileTypes = { }, ], }, - + // Links and References - for link validation { name: "checkExternalXref", @@ -187,24 +180,20 @@ let defaultFileTypes = { }, { name: "checkHyperlink", - regex: [ - ']*>', - ], + regex: [']*>'], actions: ["checkLink"], }, { name: "checkLinkElement", - regex: [ - ']*>', - ], + regex: [']*>'], actions: ["checkLink"], }, - + // Code Execution { name: "runShellCodeblock", regex: [ - "]*outputclass=\"(?:shell|bash)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + ']*outputclass="(?:shell|bash)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -217,7 +206,7 @@ let defaultFileTypes = { { name: "runCode", regex: [ - "]*outputclass=\"(python|py|javascript|js)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + ']*outputclass="(python|py|javascript|js)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -231,7 +220,7 @@ let defaultFileTypes = { }, ], }, - + // Legacy patterns for compatibility with existing tests { name: "clickOnscreenText", @@ -255,7 +244,10 @@ let defaultFileTypes = { { name: "screenshotImage", regex: [ + ']*outputclass="[^"]*screenshot[^"]*"[^>]*href="([^"]+)"[^>]*\\/>', ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[^>]*\\/>', + ']*outputclass="[^"]*screenshot[^"]*"[^>]*href="([^"]+)"[\\s\\S]*?<\\/image>', + ']*href="([^"]+)"[^>]*outputclass="[^"]*screenshot[^"]*"[\\s\\S]*?<\\/image>', ], actions: ["screenshot"], }, @@ -267,7 +259,7 @@ let defaultFileTypes = { { name: "httpRequestFormat", regex: [ - "]*outputclass=\"http\"[^>]*>\\s*([A-Z]+)\\s+([^\\s]+)(?:\\s+HTTP\\/[\\d.]+)?\\s*(?:\\r?\\n| )((?:[^\\s<]+:\\s+[^\\r\\n<]+(?:\\r?\\n| ))*)(?:\\s*(?:\\r?\\n| )([\\s\\S]*?))?\\s*<\\/codeblock>", + ']*outputclass="http"[^>]*>\\s*([A-Z]+)\\s+([^\\s]+)(?:\\s+HTTP\\/[\\d.]+)?\\s*(?:\\r?\\n| )((?:[^\\s<]+:\\s+[^\\r\\n<]+(?:\\r?\\n| ))*)(?:\\s*(?:\\r?\\n| )([\\s\\S]*?))?\\s*<\\/codeblock>', ], actions: [ { @@ -285,7 +277,7 @@ let defaultFileTypes = { { name: "runCode", regex: [ - "]*outputclass=\"(bash|python|py|javascript|js)\"[^>]*>([\\s\\S]*?)<\\/codeblock>", + ']*outputclass="(bash|python|py|javascript|js)"[^>]*>([\\s\\S]*?)<\\/codeblock>', ], actions: [ { @@ -462,7 +454,7 @@ defaultFileTypes = { /** * Resolves the concurrentRunners configuration value from various input formats * to a concrete integer for the core execution engine. - * + * * @param {Object} config - The configuration object * @returns {number} The resolved concurrent runners value */ @@ -665,10 +657,10 @@ async function setConfig({ config }) { // Detect current environment. config.environment = getEnvironment(); - + // Resolve concurrent runners configuration config.concurrentRunners = resolveConcurrentRunners(config); - + // TODO: Revise loadDescriptions() so it doesn't mutate the input but instead returns an updated object await loadDescriptions(config); diff --git a/src/heretto.integration.test.js b/src/heretto.integration.test.js new file mode 100644 index 0000000..f3a7919 --- /dev/null +++ b/src/heretto.integration.test.js @@ -0,0 +1,263 @@ +/** + * Heretto Integration Tests + * + * These tests run against the real Heretto API and are designed to only + * execute in CI environments (GitHub Actions) where credentials are available. + * + * Required environment variables: + * - HERETTO_ORGANIZATION_ID: The Heretto organization ID + * - HERETTO_USERNAME: The Heretto username (email) + * - HERETTO_API_TOKEN: The Heretto API token + * + * These tests are skipped when: + * - Running locally without CI=true environment variable + * - Required environment variables are not set + */ + +const heretto = require("./heretto"); +const fs = require("fs"); +const path = require("path"); +const os = require("os"); + +before(async function () { + const { expect } = await import("chai"); + global.expect = expect; +}); + +/** + * Check if we're running in CI and have required credentials + */ +const isCI = process.env.CI === "true"; +const hasCredentials = + process.env.HERETTO_ORGANIZATION_ID && + process.env.HERETTO_USERNAME && + process.env.HERETTO_API_TOKEN; + +const shouldRunIntegrationTests = isCI && hasCredentials; + +// Helper to skip tests when not in CI or missing credentials +const describeIntegration = shouldRunIntegrationTests ? describe : describe.skip; + +// Log why tests are being skipped +if (!shouldRunIntegrationTests) { + console.log("\n⏭️ Heretto integration tests skipped:"); + if (!isCI) { + console.log(" - Not running in CI environment (CI !== 'true')"); + } + if (!hasCredentials) { + console.log(" - Missing required environment variables:"); + if (!process.env.HERETTO_ORGANIZATION_ID) + console.log(" - HERETTO_ORGANIZATION_ID"); + if (!process.env.HERETTO_USERNAME) console.log(" - HERETTO_USERNAME"); + if (!process.env.HERETTO_API_TOKEN) console.log(" - HERETTO_API_TOKEN"); + } + console.log(""); +} + +describeIntegration("Heretto Integration Tests (CI Only)", function () { + // These tests interact with real APIs, so allow longer timeouts + this.timeout(120000); // 2 minutes per test + + let client; + let herettoConfig; + let tempDirectories = []; // Track temp directories for cleanup + const mockLog = (...args) => { + if (process.env.DEBUG) { + console.log(...args); + } + }; + const mockConfig = { logLevel: process.env.DEBUG ? "debug" : "info" }; + + before(function () { + herettoConfig = { + name: "integration-test", + organizationId: process.env.HERETTO_ORGANIZATION_ID, + username: process.env.HERETTO_USERNAME, + apiToken: process.env.HERETTO_API_TOKEN, + scenarioName: process.env.HERETTO_SCENARIO_NAME || "Doc Detective", + }; + + client = heretto.createApiClient(herettoConfig); + }); + + after(function () { + // Clean up any temporary directories created during tests + const tempDir = path.join(os.tmpdir(), "doc-detective"); + if (fs.existsSync(tempDir)) { + try { + // Find and remove heretto_* directories created during this test run + const items = fs.readdirSync(tempDir); + for (const item of items) { + if (item.startsWith("heretto_")) { + const itemPath = path.join(tempDir, item); + if (fs.statSync(itemPath).isDirectory()) { + fs.rmSync(itemPath, { recursive: true, force: true }); + if (process.env.DEBUG) { + console.log(`Cleaned up temp directory: ${itemPath}`); + } + } + } + } + } catch (error) { + // Ignore cleanup errors - these are best-effort + if (process.env.DEBUG) { + console.log(`Cleanup warning: ${error.message}`); + } + } + } + }); + + describe("API Client Creation", function () { + it("should create a valid API client", function () { + expect(client).to.not.be.null; + expect(client).to.have.property("get"); + expect(client).to.have.property("post"); + }); + + it("should configure correct base URL", function () { + const expectedBaseUrl = `https://${herettoConfig.organizationId}.heretto.com/ezdnxtgen/api/v2`; + expect(client.defaults.baseURL).to.equal(expectedBaseUrl); + }); + }); + + describe("findScenario", function () { + it("should find an existing scenario with correct configuration", async function () { + const result = await heretto.findScenario( + client, + mockLog, + mockConfig, + herettoConfig.scenarioName + ); + + // The scenario should exist and have required properties + expect(result).to.not.be.null; + expect(result).to.have.property("scenarioId"); + expect(result).to.have.property("fileId"); + expect(result.scenarioId).to.be.a("string"); + expect(result.fileId).to.be.a("string"); + }); + + it("should return null for non-existent scenario", async function () { + const result = await heretto.findScenario( + client, + mockLog, + mockConfig, + "NonExistent Scenario That Should Not Exist 12345" + ); + + expect(result).to.be.null; + }); + }); + + describe("Full Publishing Workflow", function () { + let scenarioInfo; + let jobId; + + before(async function () { + // Find the scenario first + scenarioInfo = await heretto.findScenario( + client, + mockLog, + mockConfig, + herettoConfig.scenarioName + ); + + if (!scenarioInfo) { + this.skip(); + } + }); + + it("should trigger a publishing job", async function () { + const job = await heretto.triggerPublishingJob( + client, + scenarioInfo.fileId, + scenarioInfo.scenarioId + ); + + expect(job).to.not.be.null; + expect(job).to.have.property("id"); + jobId = job.id; + }); + + it("should poll job status until completion", async function () { + // This test may take a while as it waits for the job to complete + this.timeout(360000); // 6 minutes + + const completedJob = await heretto.pollJobStatus( + client, + scenarioInfo.fileId, + jobId, + mockLog, + mockConfig + ); + + expect(completedJob).to.not.be.null; + expect(completedJob).to.have.property("status"); + expect(completedJob.status).to.have.property("status"); + + // Job should be in a completed state + const completedStates = ["COMPLETED", "FAILED", "DONE"]; + expect(completedStates).to.include(completedJob.status.status); + }); + + it("should fetch job asset details", async function () { + const assets = await heretto.getJobAssetDetails( + client, + scenarioInfo.fileId, + jobId + ); + + expect(assets).to.be.an("array"); + expect(assets.length).to.be.greaterThan(0); + + // Should contain at least some DITA files + const hasDitaFiles = assets.some( + (path) => path.endsWith(".dita") || path.endsWith(".ditamap") + ); + expect(hasDitaFiles).to.be.true; + }); + + it("should validate ditamap exists in assets", async function () { + const assets = await heretto.getJobAssetDetails( + client, + scenarioInfo.fileId, + jobId + ); + + const hasValidDitamap = heretto.validateDitamapInAssets(assets); + expect(hasValidDitamap).to.be.true; + }); + + it("should download and extract output", async function () { + const outputPath = await heretto.downloadAndExtractOutput( + client, + scenarioInfo.fileId, + jobId, + herettoConfig.name, + mockLog, + mockConfig + ); + + expect(outputPath).to.not.be.null; + expect(outputPath).to.be.a("string"); + expect(outputPath).to.include("heretto_"); + }); + }); + + describe("loadHerettoContent (End-to-End)", function () { + it("should load content from Heretto successfully", async function () { + // This is the full end-to-end test + this.timeout(600000); // 10 minutes for full workflow + + const outputPath = await heretto.loadHerettoContent( + herettoConfig, + mockLog, + mockConfig + ); + + expect(outputPath).to.not.be.null; + expect(outputPath).to.be.a("string"); + expect(outputPath).to.include("heretto_"); + }); + }); +}); diff --git a/src/heretto.js b/src/heretto.js index c535e33..be6f7d8 100644 --- a/src/heretto.js +++ b/src/heretto.js @@ -4,6 +4,7 @@ const path = require("path"); const os = require("os"); const crypto = require("crypto"); const AdmZip = require("adm-zip"); +const { XMLParser } = require("fast-xml-parser"); // Internal constants - not exposed to users const POLLING_INTERVAL_MS = 5000; @@ -11,6 +12,8 @@ const POLLING_TIMEOUT_MS = 300000; // 5 minutes const API_REQUEST_TIMEOUT_MS = 30000; // 30 seconds for individual API requests const DOWNLOAD_TIMEOUT_MS = 300000; // 5 minutes for downloads const DEFAULT_SCENARIO_NAME = "Doc Detective"; +// Base URL for REST API (different from publishing API) +const REST_API_PATH = "/rest/all-files"; /** * Creates a Base64-encoded Basic Auth header from username and API token. @@ -52,6 +55,26 @@ function createApiClient(herettoConfig) { }); } +/** + * Creates an axios instance configured for Heretto REST API requests (different base URL). + * @param {Object} herettoConfig - Heretto integration configuration + * @returns {Object} Configured axios instance for REST API + */ +function createRestApiClient(herettoConfig) { + const authHeader = createAuthHeader( + herettoConfig.username, + herettoConfig.apiToken + ); + return axios.create({ + baseURL: `https://${herettoConfig.organizationId}.heretto.com`, + timeout: API_REQUEST_TIMEOUT_MS, + headers: { + Authorization: `Basic ${authHeader}`, + Accept: "application/xml, text/xml, */*", + }, + }); +} + /** * Fetches all available publishing scenarios from Heretto. * @param {Object} client - Configured axios instance @@ -151,7 +174,10 @@ async function findScenario(client, log, config, scenarioName) { "debug", `Found existing "${scenarioName}" scenario: ${foundScenario.id}` ); - return { scenarioId: foundScenario.id, fileId: fileUuidPickerParam.value }; + return { + scenarioId: foundScenario.id, + fileId: fileUuidPickerParam.value, + }; } catch (error) { log( config, @@ -191,8 +217,64 @@ async function getJobStatus(client, fileId, jobId) { return response.data; } +/** + * Gets all asset file paths from a completed publishing job. + * Handles pagination to retrieve all assets. + * @param {Object} client - Configured axios instance + * @param {string} fileId - UUID of the DITA map + * @param {string} jobId - ID of the publishing job + * @returns {Promise>} Array of asset file paths + */ +async function getJobAssetDetails(client, fileId, jobId) { + const allAssets = []; + let page = 0; + const pageSize = 100; + let hasMorePages = true; + + while (hasMorePages) { + const response = await client.get( + `/files/${fileId}/publishes/${jobId}/assets`, + { + params: { + page, + size: pageSize, + }, + } + ); + + const data = response.data; + const content = data.content || []; + + for (const asset of content) { + if (asset.filePath) { + allAssets.push(asset.filePath); + } + } + + // Check if there are more pages + const totalPages = data.totalPages || 1; + page++; + hasMorePages = page < totalPages; + } + + return allAssets; +} + +/** + * Validates that a .ditamap file exists in the job assets. + * Checks for any .ditamap file in the ot-output/dita/ directory. + * @param {Array} assets - Array of asset file paths + * @returns {boolean} True if a .ditamap is found in ot-output/dita/ + */ +function validateDitamapInAssets(assets) { + return assets.some((assetPath) => + assetPath.startsWith("ot-output/dita/") && assetPath.endsWith(".ditamap") + ); +} + /** * Polls a publishing job until completion or timeout. + * After job completes, validates that a .ditamap file exists in the output. * @param {Object} client - Configured axios instance * @param {string} fileId - UUID of the DITA map * @param {string} jobId - ID of the publishing job @@ -208,17 +290,46 @@ async function pollJobStatus(client, fileId, jobId, log, config) { const job = await getJobStatus(client, fileId, jobId); log(config, "debug", `Job ${jobId} status: ${job?.status?.status}`); - if (job?.status?.result === "SUCCESS") { - return job; - } - - if (job?.status?.result === "FAIL") { + // Check if job has reached a terminal state (result is set) + if (job?.status?.result) { log( config, - "warning", - `Publishing job ${jobId} failed.` + "debug", + `Job ${jobId} completed with result: ${job.status.result}` ); - return null; + + // Validate that a .ditamap file exists in the output + try { + const assets = await getJobAssetDetails(client, fileId, jobId); + log( + config, + "debug", + `Job ${jobId} has ${assets.length} assets` + ); + + if (validateDitamapInAssets(assets)) { + log( + config, + "debug", + `Found .ditamap file in ot-output/dita/` + ); + return job; + } + + log( + config, + "warning", + `Publishing job ${jobId} completed but no .ditamap file found in ot-output/dita/` + ); + return null; + } catch (assetError) { + log( + config, + "warning", + `Failed to validate job assets: ${assetError.message}` + ); + return null; + } } // Wait before next poll @@ -333,6 +444,139 @@ async function downloadAndExtractOutput( } } +/** + * Retrieves resource dependencies (all files) for a ditamap from Heretto REST API. + * This provides the complete file structure with UUIDs and paths. + * @param {Object} restClient - Configured axios instance for REST API + * @param {string} ditamapId - UUID of the ditamap file + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Object mapping relative paths to UUIDs and parent folder info + */ +async function getResourceDependencies(restClient, ditamapId, log, config) { + const pathToUuidMap = {}; + + const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + }); + + // First, try to get the ditamap's own info (this is more reliable than the dependencies endpoint) + try { + log(config, "debug", `Fetching ditamap info for: ${ditamapId}`); + const ditamapInfo = await restClient.get(`${REST_API_PATH}/${ditamapId}`); + const ditamapParsed = xmlParser.parse(ditamapInfo.data); + + const ditamapUri = ditamapParsed.resource?.["xmldb-uri"] || ditamapParsed["@_uri"]; + const ditamapName = ditamapParsed.resource?.name || ditamapParsed["@_name"]; + const ditamapParentFolder = ditamapParsed.resource?.["folder-uuid"] || + ditamapParsed.resource?.["@_folder-uuid"] || + ditamapParsed["@_folder-uuid"]; + + log(config, "debug", `Ditamap info: uri=${ditamapUri}, name=${ditamapName}, parentFolder=${ditamapParentFolder}`); + + if (ditamapUri) { + let relativePath = ditamapUri; + const orgPathMatch = relativePath?.match(/\/db\/organizations\/[^/]+\/(.+)/); + if (orgPathMatch) { + relativePath = orgPathMatch[1]; + } + + pathToUuidMap[relativePath] = { + uuid: ditamapId, + fullPath: ditamapUri, + name: ditamapName, + parentFolderId: ditamapParentFolder, + isDitamap: true, + }; + + // Store the ditamap info as reference points for creating new files + pathToUuidMap._ditamapPath = relativePath; + pathToUuidMap._ditamapId = ditamapId; + pathToUuidMap._ditamapParentFolderId = ditamapParentFolder; + + log(config, "debug", `Ditamap path: ${relativePath}, parent folder: ${ditamapParentFolder}`); + } + } catch (ditamapError) { + log(config, "warning", `Could not get ditamap info: ${ditamapError.message}`); + } + + // Then try to get the full dependencies list (this endpoint may not be available) + try { + log(config, "debug", `Fetching resource dependencies for ditamap: ${ditamapId}`); + + const response = await restClient.get(`${REST_API_PATH}/${ditamapId}/dependencies`); + const xmlData = response.data; + + const parsed = xmlParser.parse(xmlData); + + // Extract dependencies from the response + // Response format: ...... + const extractDependencies = (obj, parentPath = "") => { + if (!obj) return; + + // Handle single dependency or array of dependencies + let dependencies = obj.dependencies?.dependency || obj.dependency; + if (!dependencies) { + // Try to extract from root-level response + if (obj["@_id"] && obj["@_uri"]) { + dependencies = [obj]; + } else if (Array.isArray(obj)) { + dependencies = obj; + } + } + + if (!dependencies) return; + if (!Array.isArray(dependencies)) { + dependencies = [dependencies]; + } + + for (const dep of dependencies) { + const uuid = dep["@_id"] || dep["@_uuid"] || dep.id || dep.uuid; + const uri = dep["@_uri"] || dep["@_path"] || dep.uri || dep.path || dep["xmldb-uri"]; + const name = dep["@_name"] || dep.name; + const parentFolderId = dep["@_folder-uuid"] || dep["@_parent"] || dep["folder-uuid"]; + + if (uuid && (uri || name)) { + // Extract the relative path from the full URI + // URI format: /db/organizations/{org}/{path} + let relativePath = uri || name; + const orgPathMatch = relativePath?.match(/\/db\/organizations\/[^/]+\/(.+)/); + if (orgPathMatch) { + relativePath = orgPathMatch[1]; + } + + pathToUuidMap[relativePath] = { + uuid, + fullPath: uri, + name: name || path.basename(relativePath || ""), + parentFolderId, + }; + + log(config, "debug", `Mapped: ${relativePath} -> ${uuid}`); + } + + // Recursively process nested dependencies + if (dep.dependencies || dep.dependency) { + extractDependencies(dep); + } + } + }; + + extractDependencies(parsed); + + log(config, "info", `Retrieved ${Object.keys(pathToUuidMap).length} resource dependencies from Heretto`); + + } catch (error) { + // Log more details about the error for debugging + const statusCode = error.response?.status; + log(config, "debug", `Dependencies endpoint not available (${statusCode}), will use ditamap info as fallback`); + // Continue with ditamap info only - the fallback will create files in the ditamap's parent folder + } + + return pathToUuidMap; +} + /** * Main function to load content from a Heretto CMS instance. * Triggers a publishing job, waits for completion, and downloads the output. @@ -350,10 +594,16 @@ async function loadHerettoContent(herettoConfig, log, config) { try { const client = createApiClient(herettoConfig); + const restClient = createRestApiClient(herettoConfig); // Find the Doc Detective publishing scenario const scenarioName = herettoConfig.scenarioName || DEFAULT_SCENARIO_NAME; - const scenario = await findScenario(client, log, config, scenarioName); + const scenario = await findScenario( + client, + log, + config, + scenarioName + ); if (!scenario) { log( config, @@ -363,6 +613,19 @@ async function loadHerettoContent(herettoConfig, log, config) { return null; } + // Fetch resource dependencies to build path-to-UUID mapping + // This gives us the complete file structure with UUIDs before we even run the job + if (herettoConfig.uploadOnChange) { + log(config, "debug", `Fetching resource dependencies for ditamap ${scenario.fileId}...`); + const resourceDependencies = await getResourceDependencies( + restClient, + scenario.fileId, + log, + config + ); + herettoConfig.resourceDependencies = resourceDependencies; + } + // Trigger publishing job log( config, @@ -404,6 +667,17 @@ async function loadHerettoContent(herettoConfig, log, config) { config ); + // Build file mapping from extracted content (legacy approach, still useful as fallback) + if (outputPath && herettoConfig.uploadOnChange) { + const fileMapping = await buildFileMapping( + outputPath, + herettoConfig, + log, + config + ); + herettoConfig.fileMapping = fileMapping; + } + return outputPath; } catch (error) { log( @@ -415,14 +689,371 @@ async function loadHerettoContent(herettoConfig, log, config) { } } +/** + * Builds a mapping of local file paths to Heretto file metadata. + * Parses DITA files to extract file references and attempts to resolve UUIDs. + * @param {string} outputPath - Path to extracted Heretto content + * @param {Object} herettoConfig - Heretto integration configuration + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Mapping of local paths to {fileId, filePath} + */ +async function buildFileMapping(outputPath, herettoConfig, log, config) { + const fileMapping = {}; + const xmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + }); + + try { + // Recursively find all DITA/XML files + const ditaFiles = findFilesWithExtensions(outputPath, [ + ".dita", + ".ditamap", + ".xml", + ]); + + for (const ditaFile of ditaFiles) { + try { + const content = fs.readFileSync(ditaFile, "utf-8"); + const parsed = xmlParser.parse(content); + + // Extract image references from DITA content + const imageRefs = extractImageReferences(parsed); + + for (const imageRef of imageRefs) { + // Resolve relative path to absolute local path + const absoluteLocalPath = path.resolve( + path.dirname(ditaFile), + imageRef + ); + + if (!fileMapping[absoluteLocalPath]) { + fileMapping[absoluteLocalPath] = { + filePath: imageRef, + sourceFile: ditaFile, + }; + } + } + } catch (parseError) { + log( + config, + "debug", + `Failed to parse ${ditaFile} for file mapping: ${parseError.message}` + ); + } + } + + log( + config, + "debug", + `Built file mapping with ${Object.keys(fileMapping).length} entries` + ); + } catch (error) { + log(config, "warning", `Failed to build file mapping: ${error.message}`); + } + + return fileMapping; +} + +/** + * Recursively finds files with specified extensions. + * @param {string} dir - Directory to search + * @param {Array} extensions - File extensions to match (e.g., ['.dita', '.xml']) + * @returns {Array} Array of matching file paths + */ +function findFilesWithExtensions(dir, extensions) { + const results = []; + + try { + const items = fs.readdirSync(dir); + + for (const item of items) { + const fullPath = path.join(dir, item); + const stat = fs.statSync(fullPath); + + if (stat.isDirectory()) { + results.push(...findFilesWithExtensions(fullPath, extensions)); + } else if ( + extensions.some((ext) => fullPath.toLowerCase().endsWith(ext)) + ) { + results.push(fullPath); + } + } + } catch (error) { + // Ignore read errors for inaccessible directories + } + + return results; +} + +/** + * Extracts image references from parsed DITA XML content. + * Looks for elements with href attributes. + * @param {Object} parsedXml - Parsed XML object + * @returns {Array} Array of image href values + */ +function extractImageReferences(parsedXml) { + const refs = []; + + function traverse(obj) { + if (!obj || typeof obj !== "object") return; + + // Check for image elements + if (obj.image) { + const images = Array.isArray(obj.image) ? obj.image : [obj.image]; + for (const img of images) { + if (img["@_href"]) { + refs.push(img["@_href"]); + } + } + } + + // Recursively traverse all properties + for (const key of Object.keys(obj)) { + if (typeof obj[key] === "object") { + traverse(obj[key]); + } + } + } + + traverse(parsedXml); + return refs; +} + +/** + * Searches for a file in Heretto by filename. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {string} filename - Name of the file to search for + * @param {string} folderPath - Optional folder path to search within + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} File info with ID and URI, or null if not found + */ +async function searchFileByName( + herettoConfig, + filename, + folderPath, + log, + config +) { + const client = createApiClient(herettoConfig); + + try { + const searchBody = { + queryString: filename, + foldersToSearch: {}, + startOffset: 0, + endOffset: 10, + searchResultType: "FILES_ONLY", + addPrefixAndFuzzy: false, + }; + + // If folderPath provided, search within that folder; otherwise search root + if (folderPath) { + searchBody.foldersToSearch[folderPath] = true; + } else { + // Search in organization root + searchBody.foldersToSearch[ + `/db/organizations/${herettoConfig.organizationId}/` + ] = true; + } + + const response = await client.post( + "/ezdnxtgen/api/search", + searchBody, + { + baseURL: `https://${herettoConfig.organizationId}.heretto.com`, + headers: { "Content-Type": "application/json" }, + } + ); + + if (response.data?.hits?.length > 0) { + // Find exact filename match + const exactMatch = response.data.hits.find( + (hit) => hit.fileEntity?.name === filename + ); + + if (exactMatch) { + return { + fileId: exactMatch.fileEntity.ID, + filePath: exactMatch.fileEntity.URI, + name: exactMatch.fileEntity.name, + }; + } + } + + return null; + } catch (error) { + log( + config, + "debug", + `Failed to search for file "${filename}": ${error.message}` + ); + return null; + } +} + +/** + * Uploads a file to Heretto CMS. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {string} fileId - UUID of the file to update + * @param {string} localFilePath - Local path to the file to upload + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Result object with status and description + */ +async function uploadFile(herettoConfig, fileId, localFilePath, log, config) { + const client = createRestApiClient(herettoConfig); + + try { + // Ensure the local file exists before attempting to read it + if (!fs.existsSync(localFilePath)) { + log( + config, + "warning", + `Local file does not exist, cannot upload to Heretto: ${localFilePath}` + ); + return { + status: "FAIL", + description: `Local file not found: ${localFilePath}`, + }; + } + + // Read file as binary + const fileBuffer = fs.readFileSync(localFilePath); + + // Determine content type from file extension + const ext = path.extname(localFilePath).toLowerCase(); + let contentType = "application/octet-stream"; + if (ext === ".png") contentType = "image/png"; + else if (ext === ".jpg" || ext === ".jpeg") contentType = "image/jpeg"; + else if (ext === ".gif") contentType = "image/gif"; + else if (ext === ".svg") contentType = "image/svg+xml"; + else if (ext === ".webp") contentType = "image/webp"; + + log(config, "debug", `Uploading ${localFilePath} to Heretto file ${fileId}`); + + const response = await client.put( + `${REST_API_PATH}/${fileId}/content`, + fileBuffer, + { + headers: { + "Content-Type": contentType, + "Content-Length": fileBuffer.length, + }, + maxBodyLength: Infinity, + maxContentLength: Infinity, + } + ); + + if (response.status === 200 || response.status === 201) { + log( + config, + "info", + `Successfully uploaded ${path.basename(localFilePath)} to Heretto` + ); + return { + status: "PASS", + description: `File uploaded successfully to Heretto`, + }; + } + + return { + status: "FAIL", + description: `Unexpected response status: ${response.status}`, + }; + } catch (error) { + const errorMessage = error.response?.data || error.message; + log( + config, + "warning", + `Failed to upload file to Heretto: ${errorMessage}` + ); + return { + status: "FAIL", + description: `Failed to upload: ${errorMessage}`, + }; + } +} + +/** + * Resolves a local file path to a Heretto file ID. + * First checks file mapping, then searches by filename if needed. + * @param {Object} herettoConfig - Heretto integration configuration + * @param {string} localFilePath - Local path to the file + * @param {Object} sourceIntegration - Source integration metadata from step + * @param {Function} log - Logging function + * @param {Object} config - Doc Detective config for logging + * @returns {Promise} Heretto file ID or null if not found + */ +async function resolveFileId( + herettoConfig, + localFilePath, + sourceIntegration, + log, + config +) { + // If fileId is already known, use it + if (sourceIntegration?.fileId) { + return sourceIntegration.fileId; + } + + // Check file mapping + if (herettoConfig.fileMapping && herettoConfig.fileMapping[localFilePath]) { + const mapping = herettoConfig.fileMapping[localFilePath]; + if (mapping.fileId) { + return mapping.fileId; + } + } + + // Search by filename + const filename = path.basename(localFilePath); + const searchResult = await searchFileByName( + herettoConfig, + filename, + null, + log, + config + ); + + if (searchResult?.fileId) { + // Cache the result in file mapping + if (!herettoConfig.fileMapping) { + herettoConfig.fileMapping = {}; + } + herettoConfig.fileMapping[localFilePath] = { + fileId: searchResult.fileId, + filePath: searchResult.filePath, + }; + return searchResult.fileId; + } + + log( + config, + "warning", + `Could not resolve Heretto file ID for ${localFilePath}` + ); + return null; +} + module.exports = { createAuthHeader, createApiClient, + createRestApiClient, findScenario, triggerPublishingJob, + getJobStatus, + getJobAssetDetails, + validateDitamapInAssets, pollJobStatus, downloadAndExtractOutput, loadHerettoContent, + buildFileMapping, + searchFileByName, + uploadFile, + resolveFileId, + getResourceDependencies, // Export constants for testing POLLING_INTERVAL_MS, POLLING_TIMEOUT_MS, diff --git a/src/heretto.test.js b/src/heretto.test.js index c89ea82..c47a5a8 100644 --- a/src/heretto.test.js +++ b/src/heretto.test.js @@ -100,9 +100,11 @@ describe("Heretto Integration", function () { const result = await heretto.findScenario(mockClient, mockLog, mockConfig, "Doc Detective"); - expect(result).to.deep.equal({ scenarioId: "scenario-123", fileId: "file-uuid-456" }); + expect(result).to.deep.equal({ + scenarioId: "scenario-123", + fileId: "file-uuid-456", + }); expect(mockClient.get.calledTwice).to.be.true; - expect(mockClient.post.called).to.be.false; }); it("should return null if scenario is not found", async function () { @@ -114,7 +116,6 @@ describe("Heretto Integration", function () { expect(result).to.be.null; expect(mockClient.get.calledOnce).to.be.true; - expect(mockClient.post.called).to.be.false; }); it("should return null if scenario fetch fails", async function () { @@ -228,6 +229,157 @@ describe("Heretto Integration", function () { }); }); + describe("getJobAssetDetails", function () { + it("should return all asset file paths from single page", async function () { + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + { filePath: "ot-output/dita/topic1.dita" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 1, + number: 0, + size: 100, + }; + + mockClient.get.resolves({ data: assetsResponse }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([ + "ot-output/dita/my-guide.ditamap", + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + ]); + expect(mockClient.get.calledOnce).to.be.true; + expect(mockClient.get.firstCall.args[0]).to.equal("/files/file-uuid/publishes/job-123/assets"); + }); + + it("should handle pagination and aggregate all assets", async function () { + const page1Response = { + content: [ + { filePath: "ot-output/dita/topic1.dita" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 2, + number: 0, + size: 100, + }; + + const page2Response = { + content: [ + { filePath: "ot-output/dita/topic3.dita" }, + { filePath: "ot-output/dita/my-guide.ditamap" }, + ], + totalPages: 2, + number: 1, + size: 100, + }; + + mockClient.get + .onFirstCall().resolves({ data: page1Response }) + .onSecondCall().resolves({ data: page2Response }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([ + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + "ot-output/dita/topic3.dita", + "ot-output/dita/my-guide.ditamap", + ]); + expect(mockClient.get.calledTwice).to.be.true; + }); + + it("should return empty array when no assets", async function () { + const assetsResponse = { + content: [], + totalPages: 1, + number: 0, + size: 100, + }; + + mockClient.get.resolves({ data: assetsResponse }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([]); + }); + + it("should skip assets without filePath", async function () { + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/topic1.dita" }, + { otherField: "no-path" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 1, + }; + + mockClient.get.resolves({ data: assetsResponse }); + + const result = await heretto.getJobAssetDetails(mockClient, "file-uuid", "job-123"); + + expect(result).to.deep.equal([ + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + ]); + }); + }); + + describe("validateDitamapInAssets", function () { + it("should return true when ditamap is in ot-output/dita/", function () { + const assets = [ + "ot-output/dita/topic1.dita", + "ot-output/dita/my-guide.ditamap", + "ot-output/dita/topic2.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.true; + }); + + it("should return false when no ditamap is present", function () { + const assets = [ + "ot-output/dita/topic1.dita", + "ot-output/dita/topic2.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.false; + }); + + it("should return false when ditamap is in wrong directory", function () { + const assets = [ + "ot-output/other/my-guide.ditamap", + "ot-output/dita/topic1.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.false; + }); + + it("should return true when any ditamap is in correct directory", function () { + const assets = [ + "ot-output/dita/different-guide.ditamap", + "ot-output/dita/topic1.dita", + ]; + + const result = heretto.validateDitamapInAssets(assets); + + expect(result).to.be.true; + }); + + it("should return false when assets array is empty", function () { + const result = heretto.validateDitamapInAssets([]); + + expect(result).to.be.false; + }); + }); + describe("pollJobStatus", function () { const mockLog = sinon.stub(); const mockConfig = { logLevel: "info" }; @@ -236,40 +388,91 @@ describe("Heretto Integration", function () { mockLog.reset(); }); - it("should return completed job when status.result is SUCCESS", async function () { + it("should return completed job when status.result is SUCCESS and ditamap is present", async function () { const completedJob = { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" }, }; - mockClient.get.resolves({ data: completedJob }); + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + { filePath: "ot-output/dita/topic1.dita" }, + ], + totalPages: 1, + }; + + mockClient.get + .onFirstCall().resolves({ data: completedJob }) + .onSecondCall().resolves({ data: assetsResponse }); const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); expect(result).to.deep.equal(completedJob); }); - it("should return null when status.result is FAIL", async function () { + it("should return completed job when status.result is FAIL but ditamap is present", async function () { const failedJob = { id: "job-123", status: { status: "FAILED", result: "FAIL" }, }; - mockClient.get.resolves({ data: failedJob }); + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + { filePath: "ot-output/dita/topic1.dita" }, + ], + totalPages: 1, + }; + + mockClient.get + .onFirstCall().resolves({ data: failedJob }) + .onSecondCall().resolves({ data: assetsResponse }); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.deep.equal(failedJob); + }); + + it("should return null when job completes but ditamap is missing", async function () { + const completedJob = { + id: "job-123", + status: { status: "COMPLETED", result: "SUCCESS" }, + }; + + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/topic1.dita" }, + { filePath: "ot-output/dita/topic2.dita" }, + ], + totalPages: 1, + }; + + mockClient.get + .onFirstCall().resolves({ data: completedJob }) + .onSecondCall().resolves({ data: assetsResponse }); const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); expect(result).to.be.null; }); - it("should poll until completion", async function () { + it("should poll until completion then validate assets", async function () { // Use fake timers to avoid waiting for real POLLING_INTERVAL_MS delays const clock = sinon.useFakeTimers(); + const assetsResponse = { + content: [ + { filePath: "ot-output/dita/my-guide.ditamap" }, + ], + totalPages: 1, + }; + mockClient.get - .onFirstCall().resolves({ data: { id: "job-123", status: { status: "PENDING", result: null } } }) - .onSecondCall().resolves({ data: { id: "job-123", status: { status: "PROCESSING", result: null } } }) - .onThirdCall().resolves({ data: { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" } } }); + .onCall(0).resolves({ data: { id: "job-123", status: { status: "PENDING", result: null } } }) + .onCall(1).resolves({ data: { id: "job-123", status: { status: "PROCESSING", result: null } } }) + .onCall(2).resolves({ data: { id: "job-123", status: { status: "COMPLETED", result: "SUCCESS" } } }) + .onCall(3).resolves({ data: assetsResponse }); const pollPromise = heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); @@ -281,7 +484,7 @@ describe("Heretto Integration", function () { const result = await pollPromise; expect(result.status.result).to.equal("SUCCESS"); - expect(mockClient.get.callCount).to.equal(3); + expect(mockClient.get.callCount).to.equal(4); // 3 status polls + 1 assets call clock.restore(); }); @@ -313,6 +516,21 @@ describe("Heretto Integration", function () { expect(result).to.be.null; }); + + it("should return null when asset validation fails", async function () { + const completedJob = { + id: "job-123", + status: { status: "COMPLETED", result: "SUCCESS" }, + }; + + mockClient.get + .onFirstCall().resolves({ data: completedJob }) + .onSecondCall().rejects(new Error("Failed to fetch assets")); + + const result = await heretto.pollJobStatus(mockClient, "file-uuid", "job-123", mockLog, mockConfig); + + expect(result).to.be.null; + }); }); describe("loadHerettoContent", function () { diff --git a/src/utils.js b/src/utils.js index 4c32bb9..8517ad3 100644 --- a/src/utils.js +++ b/src/utils.js @@ -26,6 +26,28 @@ exports.cleanTemp = cleanTemp; exports.calculatePercentageDifference = calculatePercentageDifference; exports.fetchFile = fetchFile; exports.isRelativeUrl = isRelativeUrl; +exports.findHerettoIntegration = findHerettoIntegration; + +/** + * Finds which Heretto integration a file belongs to based on its path. + * @param {Object} config - Doc Detective config with _herettoPathMapping + * @param {string} filePath - Path to check + * @returns {string|null} Heretto integration name or null if not from Heretto + */ +function findHerettoIntegration(config, filePath) { + if (!config._herettoPathMapping) return null; + + const normalizedFilePath = path.resolve(filePath); + + for (const [outputPath, integrationName] of Object.entries(config._herettoPathMapping)) { + const normalizedOutputPath = path.resolve(outputPath); + if (normalizedFilePath.startsWith(normalizedOutputPath)) { + return integrationName; + } + } + + return null; +} function isRelativeUrl(url) { try { @@ -38,6 +60,33 @@ function isRelativeUrl(url) { } } +/** + * Generates a unique specId from a file path that is safe for storage/URLs. + * Uses relative path from cwd when possible to provide uniqueness while + * avoiding collisions from files with the same basename in different directories. + * @param {string} filePath - Absolute or relative file path + * @returns {string} A safe specId derived from the file path + */ +function generateSpecId(filePath) { + const absolutePath = path.resolve(filePath); + const cwd = process.cwd(); + + let relativePath; + if (absolutePath.startsWith(cwd)) { + relativePath = path.relative(cwd, absolutePath); + } else { + relativePath = absolutePath; + } + + const normalizedPath = relativePath + .split(path.sep) + .join("/") + .replace(/^\.\//, "") + .replace(/[^a-zA-Z0-9._\-\/]/g, "_"); + + return normalizedPath; +} + // Parse XML-style attributes to an object // Example: 'wait=500' becomes { wait: 500 } // Example: 'testId="myTestId" detectSteps=false' becomes { testId: "myTestId", detectSteps: false } @@ -230,6 +279,11 @@ async function qualifyFiles({ config }) { } const ignoredDitaMaps = []; + + // Track Heretto output paths for sourceIntegration metadata + if (!config._herettoPathMapping) { + config._herettoPathMapping = {}; + } for (let source of sequence) { log(config, "debug", `source: ${source}`); @@ -256,6 +310,8 @@ async function qualifyFiles({ config }) { const outputPath = await loadHerettoContent(herettoConfig, log, config); if (outputPath) { herettoConfig.outputPath = outputPath; + // Store mapping from output path to Heretto integration name + config._herettoPathMapping[outputPath] = herettoName; log(config, "debug", `Adding Heretto output path: ${outputPath}`); // Insert the output path into the sequence for processing const currentIndex = sequence.indexOf(source); @@ -728,10 +784,47 @@ async function parseContent({ config, content, filePath, fileType }) { ) { step[action].origin = config.origin; } + // Attach sourceIntegration metadata for screenshot steps from Heretto + if (action === "screenshot" && config._herettoPathMapping) { + const herettoIntegration = findHerettoIntegration(config, filePath); + if (herettoIntegration) { + // Convert simple screenshot value to object with sourceIntegration + const screenshotPath = step[action]; + step[action] = { + path: screenshotPath, + sourceIntegration: { + type: "heretto", + integrationName: herettoIntegration, + filePath: screenshotPath, + contentPath: filePath, + }, + }; + } + } } else { // Substitute variables $n with match[n] // TODO: Make key substitution recursive step = replaceNumericVariables(action, statement); + + // Attach sourceIntegration metadata for screenshot steps from Heretto + if (step.screenshot && config._herettoPathMapping) { + const herettoIntegration = findHerettoIntegration(config, filePath); + if (herettoIntegration) { + // Ensure screenshot is an object + if (typeof step.screenshot === "string") { + step.screenshot = { path: step.screenshot }; + } else if (typeof step.screenshot === "boolean") { + step.screenshot = {}; + } + // Attach sourceIntegration + step.screenshot.sourceIntegration = { + type: "heretto", + integrationName: herettoIntegration, + filePath: step.screenshot.path || "", + contentPath: filePath, + }; + } + } } // Normalize step field formats @@ -912,7 +1005,9 @@ async function parseTests({ config, files }) { specs.push(content); } else { // Process non-object - let id = `${crypto.randomUUID()}`; + // Generate a specId that includes more of the file path to avoid collisions + // when different files share the same basename + let id = generateSpecId(file); let spec = { specId: id, contentPath: file, tests: [] }; const fileType = config.fileTypes.find((fileType) => fileType.extensions.includes(extension)