From 71e1aa254da1b83f0a55d4fc532f50a93af1845b Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 2 Oct 2025 14:54:08 -0400 Subject: [PATCH 1/2] MLE-24531 Converting Jenkinsfile to use Docker --- Jenkinsfile | 128 +++++++++++++++++-------- test-app/docker-compose-nightlies.yaml | 27 ------ test-app/docker-compose.yaml | 7 +- test-app/gradle.properties | 2 +- 4 files changed, 91 insertions(+), 73 deletions(-) delete mode 100644 test-app/docker-compose-nightlies.yaml diff --git a/Jenkinsfile b/Jenkinsfile index 7a969204..6853780c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,26 +1,49 @@ @Library('shared-libraries') _ -def runTests(String type,String version){ - copyRPM type,version - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' - sh ''' - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$JAVA_HOME/bin:$GRADLE_USER_HOME:${NODE_HOME_DIR}/bin:$PATH - cd node-client-api - node --version - npm --version - npm ci - cd test-app - ./gradlew -i mlDeploy -g $PWD - cd .. - rm -rf $WORKSPACE/*.xml || true - ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic/ --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-reports.xml -g \'logging|archivePath\' --invert || true - ./node_modules/.bin/gulp setupProxyTests || true - ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic-proxy/lib/**/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-proxy-reports.xml -g \'logging|archivePath\' --invert || true +def runDockerCompose(String markLogicDockerImage) { + cleanupDocker() + sh label:'mlsetup', script: '''#!/bin/bash + echo "Removing any running MarkLogic server and clean up MarkLogic data directory" + sudo /usr/local/sbin/mladmin remove + docker-compose down -v || true + sudo /usr/local/sbin/mladmin cleandata + cd node-client-api/test-app + MARKLOGIC_LOGS_VOLUME=/tmp MARKLOGIC_IMAGE='''+markLogicDockerImage+''' docker-compose up -d --build + sleep 30s; ''' +} +def runTests() { + sh label:'deploy ml-gradle test application', script: '''#!/bin/bash + export JAVA_HOME=$JAVA17_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH:${NODE_HOME_DIR}/bin:$PATH + cd node-client-api/test-app + ./gradlew -i mlTestConnections + ./gradlew -i mlDeploy + ''' + + sh label:'test', script: '''#!/bin/bash + cd node-client-api + node --version + npm --version + npm ci + rm -rf $WORKSPACE/*.xml || true + ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic/ --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-reports.xml -g \'logging|archivePath\' --invert || true + ./node_modules/.bin/gulp setupProxyTests || true + ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic-proxy/lib/**/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-proxy-reports.xml -g \'logging|archivePath\' --invert || true + ''' } + +def teardownAfterTests() { + updateWorkspacePermissions() + sh label:'mlcleanup', script: '''#!/bin/bash + cd node-client-api/test-app + docker-compose down -v || true + ''' + cleanupDocker() +} + def runAuditReport(){ sh ''' export PATH=${NODE_HOME_DIR}/bin:$PATH @@ -31,9 +54,7 @@ def runAuditReport(){ ''' } -def runE2ETests(String type,String version){ - copyRPM type,version - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' +def runE2ETests(){ sh ''' export PATH=${NODE_HOME_DIR}/bin:$PATH cd node-client-api @@ -71,18 +92,23 @@ def runE2ETests(String type,String version){ junit '**/*.xml' } + pipeline{ agent none + triggers{ parameterizedCron(env.BRANCH_NAME == "develop" ? "00 02 * * * % regressions=true" : "") } + parameters{ booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') } + options { checkoutToSubdirectory 'node-client-api' buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '10') } + environment{ NODE_HOME_DIR= "/users/ml/builder/nodeJs/node-v22.20.0-linux-x64" DMC_USER = credentials('MLBUILD_USER') @@ -90,15 +116,26 @@ pipeline{ GRADLE_DIR=".gradle" JAVA_HOME_DIR="/home/builder/java/jdk-17.0.2" } - stages{ - stage('runtests-11.3.1'){ + + stages { + + stage('runtests-12-nightly'){ agent {label 'nodeclientpool'} steps{ runAuditReport() - runTests('Release','11.3.1') - runE2ETests('Release','11.3.1') + // Temporarily using latest-11 to match what this stage previously did with rpm's. + // runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12') + runDockerCompose('progressofficial/marklogic-db:latest-11.3') + runTests() + runE2ETests() + } + post{ + always{ + teardownAfterTests() + } } } + stage('regressions'){ parallel{ stage('runtests-11-nightly'){ @@ -110,10 +147,17 @@ pipeline{ } agent {label 'nodeclientpool'} steps{ - runTests('Latest','11') - runE2ETests('Latest','11') + runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11') + runTests() + runE2ETests() + } + post{ + always{ + teardownAfterTests() + } } } + stage('runtests-12-nightly'){ when{ allOf{ @@ -123,10 +167,17 @@ pipeline{ } agent {label 'nodeclientpool'} steps{ - runTests('Latest','12.0') - runE2ETests('Latest','12.0') + runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12') + runTests() + runE2ETests() + } + post{ + always{ + teardownAfterTests() + } } } + stage('runtests-10-nightly'){ when{ allOf{ @@ -136,21 +187,14 @@ pipeline{ } agent {label 'nodeclientpool'} steps{ - runTests('Latest','10.0') - runE2ETests('Latest','10.0') + runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10') + runTests() + runE2ETests() } - } - stage('runtests-10.0-10.2'){ - when{ - allOf{ - branch 'develop' - expression {return params.regressions} - } + post{ + always{ + teardownAfterTests() } - agent {label 'nodeclientpool'} - steps{ - runTests('Release','10.0-10.2') - runE2ETests('Release','10.0-10.2') } } } diff --git a/test-app/docker-compose-nightlies.yaml b/test-app/docker-compose-nightlies.yaml deleted file mode 100644 index 161a0fde..00000000 --- a/test-app/docker-compose-nightlies.yaml +++ /dev/null @@ -1,27 +0,0 @@ -version: '3.8' -# name: node-client-10-nightly -# name: node-client-10_0-10 -name: node-client-11-1-jenkins -#name: node-client-11-nightly-jenkins -# name: node-client-12-nightly - -services: - - marklogic: - #image: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos:10.0-nightly-centos-1.1.1" - #image: "marklogicdb/marklogic-db:latest-10.0-10" - image: "marklogicdb/marklogic-db:11.1.0-centos-1.1.0" - # image: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos:11.2.nightly-centos-1.1.2" - #image: "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos:12.0.nightly-centos-1.1.1" - platform: linux/amd64 - environment: - - INSTALL_CONVERTERS=true - - MARKLOGIC_INIT=true - - MARKLOGIC_ADMIN_USERNAME=admin - - MARKLOGIC_ADMIN_PASSWORD=admin - volumes: - - ./containerLogs:/var/opt/MarkLogic/Logs - ports: - - 8000-8017:8000-8017 - - 8024-8029:8024-8029 - - 8079:8079 diff --git a/test-app/docker-compose.yaml b/test-app/docker-compose.yaml index 1aa79bbc..8d0f2352 100644 --- a/test-app/docker-compose.yaml +++ b/test-app/docker-compose.yaml @@ -1,4 +1,4 @@ -name: node-client +name: docker-tests-node-client services: @@ -13,5 +13,6 @@ services: volumes: - ${MARKLOGIC_LOGS_VOLUME}:/var/opt/MarkLogic/Logs ports: - - 8000-8002:8000-8002 - - 8015-8017:8015-8017 + - 8000-8017:8000-8017 + - 8024-8029:8024-8029 + - 8079:8079 diff --git a/test-app/gradle.properties b/test-app/gradle.properties index 3a31f809..b7c63bc5 100644 --- a/test-app/gradle.properties +++ b/test-app/gradle.properties @@ -4,4 +4,4 @@ mlHost=localhost mlUsername=admin mlPassword=admin mlRestPort=8015 -mlContentForestsPerHost=3 \ No newline at end of file +mlContentForestsPerHost=1 \ No newline at end of file From 8078b8048d3a6d8e1019d78e6c5f5f6753a3dfe9 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 6 Oct 2025 13:59:04 -0400 Subject: [PATCH 2/2] Temporarily reverting changes Want to see if the PR tests pass without any changes. --- Jenkinsfile | 128 ++++++++++++------------------------- test-app/gradle.properties | 2 +- 2 files changed, 43 insertions(+), 87 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6853780c..7a969204 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,49 +1,26 @@ @Library('shared-libraries') _ -def runDockerCompose(String markLogicDockerImage) { - cleanupDocker() - sh label:'mlsetup', script: '''#!/bin/bash - echo "Removing any running MarkLogic server and clean up MarkLogic data directory" - sudo /usr/local/sbin/mladmin remove - docker-compose down -v || true - sudo /usr/local/sbin/mladmin cleandata - cd node-client-api/test-app - MARKLOGIC_LOGS_VOLUME=/tmp MARKLOGIC_IMAGE='''+markLogicDockerImage+''' docker-compose up -d --build - sleep 30s; +def runTests(String type,String version){ + copyRPM type,version + setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + sh ''' + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$JAVA_HOME/bin:$GRADLE_USER_HOME:${NODE_HOME_DIR}/bin:$PATH + cd node-client-api + node --version + npm --version + npm ci + cd test-app + ./gradlew -i mlDeploy -g $PWD + cd .. + rm -rf $WORKSPACE/*.xml || true + ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic/ --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-reports.xml -g \'logging|archivePath\' --invert || true + ./node_modules/.bin/gulp setupProxyTests || true + ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic-proxy/lib/**/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-proxy-reports.xml -g \'logging|archivePath\' --invert || true ''' -} -def runTests() { - sh label:'deploy ml-gradle test application', script: '''#!/bin/bash - export JAVA_HOME=$JAVA17_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH:${NODE_HOME_DIR}/bin:$PATH - cd node-client-api/test-app - ./gradlew -i mlTestConnections - ./gradlew -i mlDeploy - ''' - - sh label:'test', script: '''#!/bin/bash - cd node-client-api - node --version - npm --version - npm ci - rm -rf $WORKSPACE/*.xml || true - ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic/ --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-reports.xml -g \'logging|archivePath\' --invert || true - ./node_modules/.bin/gulp setupProxyTests || true - ./node_modules/.bin/mocha --timeout 10000 -R xunit test-basic-proxy/lib/**/*.js --reporter mocha-junit-reporter --reporter-options mochaFile=$WORKSPACE/test-basic-proxy-reports.xml -g \'logging|archivePath\' --invert || true - ''' } - -def teardownAfterTests() { - updateWorkspacePermissions() - sh label:'mlcleanup', script: '''#!/bin/bash - cd node-client-api/test-app - docker-compose down -v || true - ''' - cleanupDocker() -} - def runAuditReport(){ sh ''' export PATH=${NODE_HOME_DIR}/bin:$PATH @@ -54,7 +31,9 @@ def runAuditReport(){ ''' } -def runE2ETests(){ +def runE2ETests(String type,String version){ + copyRPM type,version + setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' sh ''' export PATH=${NODE_HOME_DIR}/bin:$PATH cd node-client-api @@ -92,23 +71,18 @@ def runE2ETests(){ junit '**/*.xml' } - pipeline{ agent none - triggers{ parameterizedCron(env.BRANCH_NAME == "develop" ? "00 02 * * * % regressions=true" : "") } - parameters{ booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') } - options { checkoutToSubdirectory 'node-client-api' buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '7', numToKeepStr: '10') } - environment{ NODE_HOME_DIR= "/users/ml/builder/nodeJs/node-v22.20.0-linux-x64" DMC_USER = credentials('MLBUILD_USER') @@ -116,26 +90,15 @@ pipeline{ GRADLE_DIR=".gradle" JAVA_HOME_DIR="/home/builder/java/jdk-17.0.2" } - - stages { - - stage('runtests-12-nightly'){ + stages{ + stage('runtests-11.3.1'){ agent {label 'nodeclientpool'} steps{ runAuditReport() - // Temporarily using latest-11 to match what this stage previously did with rpm's. - // runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12') - runDockerCompose('progressofficial/marklogic-db:latest-11.3') - runTests() - runE2ETests() - } - post{ - always{ - teardownAfterTests() - } + runTests('Release','11.3.1') + runE2ETests('Release','11.3.1') } } - stage('regressions'){ parallel{ stage('runtests-11-nightly'){ @@ -147,17 +110,10 @@ pipeline{ } agent {label 'nodeclientpool'} steps{ - runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11') - runTests() - runE2ETests() - } - post{ - always{ - teardownAfterTests() - } + runTests('Latest','11') + runE2ETests('Latest','11') } } - stage('runtests-12-nightly'){ when{ allOf{ @@ -167,17 +123,10 @@ pipeline{ } agent {label 'nodeclientpool'} steps{ - runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12') - runTests() - runE2ETests() - } - post{ - always{ - teardownAfterTests() - } + runTests('Latest','12.0') + runE2ETests('Latest','12.0') } } - stage('runtests-10-nightly'){ when{ allOf{ @@ -187,14 +136,21 @@ pipeline{ } agent {label 'nodeclientpool'} steps{ - runDockerCompose('ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-10') - runTests() - runE2ETests() + runTests('Latest','10.0') + runE2ETests('Latest','10.0') } - post{ - always{ - teardownAfterTests() + } + stage('runtests-10.0-10.2'){ + when{ + allOf{ + branch 'develop' + expression {return params.regressions} + } } + agent {label 'nodeclientpool'} + steps{ + runTests('Release','10.0-10.2') + runE2ETests('Release','10.0-10.2') } } } diff --git a/test-app/gradle.properties b/test-app/gradle.properties index b7c63bc5..3a31f809 100644 --- a/test-app/gradle.properties +++ b/test-app/gradle.properties @@ -4,4 +4,4 @@ mlHost=localhost mlUsername=admin mlPassword=admin mlRestPort=8015 -mlContentForestsPerHost=1 \ No newline at end of file +mlContentForestsPerHost=3 \ No newline at end of file