diff --git a/.github/README.md b/.github/README.md new file mode 120000 index 000000000000..4dde5b34a128 --- /dev/null +++ b/.github/README.md @@ -0,0 +1 @@ +../README-DRUM.md \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39a6f41429fd..679543cb01aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,35 +78,37 @@ jobs: path: 'dspace/target/site/jacoco-aggregate/jacoco.xml' retention-days: 14 - # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test - # job above. This is necessary because Codecov uploads seem to randomly fail at times. - # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 - codecov: - # Must run after 'tests' job above - needs: tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 + # UMD Customization + # # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test + # # job above. This is necessary because Codecov uploads seem to randomly fail at times. + # # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 + # codecov: + # # Must run after 'tests' job above + # needs: tests + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v4 - # Download artifacts from previous 'tests' job - - name: Download coverage artifacts - uses: actions/download-artifact@v4 + # # Download artifacts from previous 'tests' job + # - name: Download coverage artifacts + # uses: actions/download-artifact@v4 - # Now attempt upload to Codecov using its action. - # NOTE: We use a retry action to retry the Codecov upload if it fails the first time. - # - # Retry action: https://github.com/marketplace/actions/retry-action - # Codecov action: https://github.com/codecov/codecov-action - - name: Upload coverage to Codecov.io - uses: Wandalen/wretry.action@v1.3.0 - with: - action: codecov/codecov-action@v4 - # Ensure codecov-action throws an error when it fails to upload - with: | - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - # Try re-running action 5 times max - attempt_limit: 5 - # Run again in 30 seconds - attempt_delay: 30000 + # # Now attempt upload to Codecov using its action. + # # NOTE: We use a retry action to retry the Codecov upload if it fails the first time. + # # + # # Retry action: https://github.com/marketplace/actions/retry-action + # # Codecov action: https://github.com/codecov/codecov-action + # - name: Upload coverage to Codecov.io + # uses: Wandalen/wretry.action@v1.3.0 + # with: + # action: codecov/codecov-action@v4 + # # Ensure codecov-action throws an error when it fails to upload + # with: | + # fail_ci_if_error: true + # token: ${{ secrets.CODECOV_TOKEN }} + # # Try re-running action 5 times max + # attempt_limit: 5 + # # Run again in 30 seconds + # attempt_delay: 30000 + # End UMD Customization diff --git a/.gitignore b/.gitignore index 529351edc5c2..0f20cfa4152b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,18 @@ +## UMD Customization +## Ignore temporary directories related to helper scripts +scripts/work +scripts/patches + +# Ignore HTTPS certificates +dspace/src/main/docker/nginx/certs/* +!dspace/src/main/docker/nginx/certs/.keep + +# Ignore database dump directory to be used with docker-compose for initializing +postgres-init/* +!postgres-init/README.md +!postgres-init/pg_restore.sh +## End UMD Customization + ## Ignore the MVN compiled output directories from version tracking target/ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000000..59eae1767210 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "java", + "name": "Debug (Attach to Tomcat)", + "request": "attach", + "hostName": "localhost", + "port": 8000, + "projectName": "server" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000000..3e622a8a9400 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "java.configuration.updateBuildConfiguration": "automatic", + "files.exclude": { + "**/.classpath": true, + "**/.project": true, + "**/.settings": true, + "**/.factorypath": true + } +} \ No newline at end of file diff --git a/DRUM-LICENSE.md b/DRUM-LICENSE.md new file mode 100644 index 000000000000..8dada3edaf50 --- /dev/null +++ b/DRUM-LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Dockerfile b/Dockerfile index eb90299ccd7d..0386481044f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,13 +7,19 @@ # To build with other versions, use "--build-arg JDK_VERSION=[value]" ARG JDK_VERSION=17 # The Docker version tag to build from -ARG DSPACE_VERSION=dspace-8_x +# UMD Customization +# Continuing to use "latest" because this allows a new image to be easily +# created and pushed to the Nexus +ARG DSPACE_VERSION=latest +# End UMD Customization # The Docker registry to use for DSpace images. Defaults to "docker.io" # NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument ARG DOCKER_REGISTRY=docker.io # Step 1 - Run Maven Build -FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build +# UMD Customization +FROM docker.lib.umd.edu/drum-dependencies-8_x:${DSPACE_VERSION} AS build +# End UMD Customization ARG TARGET_DIR=dspace-installer WORKDIR /app # The dspace-installer directory will be written to /install @@ -69,5 +75,25 @@ RUN apt-get update \ EXPOSE 8080 8000 # Give java extra memory (2GB) ENV JAVA_OPTS=-Xmx2000m + +# UMD Customization +ENV TZ=America/New_York + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + rsync \ + openssh-client \ + cron \ + csh \ + postfix \ + s-nail \ + libgetopt-complete-perl \ + libconfig-properties-perl \ + vim \ + python3-lxml \ + jq && \ + mkfifo /var/spool/postfix/public/pickup && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +# End UMD Customization # On startup, run DSpace Runnable JAR ENTRYPOINT ["java", "-jar", "webapps/server-boot.jar", "--dspace.dir=$DSPACE_INSTALL"] diff --git a/Dockerfile.ant b/Dockerfile.ant new file mode 100644 index 000000000000..a9b5c73ce4ef --- /dev/null +++ b/Dockerfile.ant @@ -0,0 +1,16 @@ +# This Docker image is based on Step 2 in "Dockerfile", with some steps removed +# as they will be performed in the Dockerfiles that use this image +# (Dockerfile.dev, Dockerfile.dev-additions) +ARG JDK_VERSION=17 + +FROM eclipse-temurin:${JDK_VERSION} AS ant_build +# Create the initial install deployment using ANT +ENV ANT_VERSION=1.10.13 +ENV ANT_HOME=/tmp/ant-$ANT_VERSION +ENV PATH=$ANT_HOME/bin:$PATH +# Download and install 'ant' +RUN mkdir $ANT_HOME && \ + curl --silent --show-error --location --fail --retry 5 --output /tmp/apache-ant.tar.gz \ + https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.tar.gz && \ + tar -zx --strip-components=1 -f /tmp/apache-ant.tar.gz -C $ANT_HOME && \ + rm /tmp/apache-ant.tar.gz diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 000000000000..413a9e4296f8 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,16 @@ +# Dockerfile for use by the continuous integration server (ci), in order to +# build and test the application. +# +# This Dockerfile provides the appropriate environment for building and testing +# the application. It should _not_ be used for creating Docker images for use +# in production. + +FROM maven:3.8.6-eclipse-temurin-17 + +# Install git, as it is needed by the Jenkinsfile +RUN apt-get update && \ + apt-get install -y build-essential && \ + apt-get install -y git && \ + apt-get clean + +COPY dspace/src/main/docker/mvn-settings.xml /usr/share/maven/ref/settings.xml diff --git a/Dockerfile.cli b/Dockerfile.cli index 1f08357b3bba..72913d5e5e9d 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -7,13 +7,19 @@ # To build with other versions, use "--build-arg JDK_VERSION=[value]" ARG JDK_VERSION=17 # The Docker version tag to build from -ARG DSPACE_VERSION=dspace-8_x +# UMD Customization +# Continuing to use "latest" because this allows a new image to be easily +# created and pushed to the Nexus +ARG DSPACE_VERSION=latest +# End UMD Customization # The Docker registry to use for DSpace images. Defaults to "docker.io" # NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument ARG DOCKER_REGISTRY=docker.io # Step 1 - Run Maven Build -FROM ${DOCKER_REGISTRY}/dspace/dspace-dependencies:${DSPACE_VERSION} AS build +# UMD Customization +FROM docker.lib.umd.edu/drum-dependencies-8_x:${DSPACE_VERSION} AS build +# End UMD Customization ARG TARGET_DIR=dspace-installer WORKDIR /app # The dspace-installer directory will be written to /install @@ -23,6 +29,10 @@ RUN mkdir /install \ USER dspace # Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents) ADD --chown=dspace . /app/ +# UMD Customization +COPY --chown=dspace dspace/src/main/docker/mvn-settings.xml /home/dspace/.m2/settings.xml +# End UMD Customization + # Build DSpace. Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small RUN mvn --no-transfer-progress package && \ mv /app/dspace/target/${TARGET_DIR}/* /install && \ diff --git a/Dockerfile.dependencies b/Dockerfile.dependencies index 04233cd415fa..9f64fd185874 100644 --- a/Dockerfile.dependencies +++ b/Dockerfile.dependencies @@ -1,4 +1,6 @@ -# This image will be published as dspace/dspace-dependencies +# UMD Customization +# This image will be published as docker.lib.umd.edu/drum-dependencies-8_x:latest +# End UMD Customization # The purpose of this image is to make the build for dspace/dspace run faster # @@ -19,6 +21,10 @@ RUN chown -Rv dspace: /app # Switch to dspace user & run below commands as that user USER dspace +# UMD Customization +# Add maven settings +COPY --chown=dspace dspace/src/main/docker/mvn-settings.xml /home/dspace/.m2/settings.xml +# End UMD Customization # This next part may look odd, but it speeds up the build of this image *significantly*. # Copy ONLY the POMs to this image (from local machine). This will allow us to download all dependencies *without* # performing any code compilation steps. diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000000..90e1e579a18c --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,94 @@ +# UMD-provided file running DSpace as part of a "Docker Compose" stack +# This file is based on the stock "Dockerfile" + +# This Dockerfile uses JDK17 by default. +# To build with other versions, use "--build-arg JDK_VERSION=[value]" +ARG JDK_VERSION=17 +# The Docker version tag to build from +# UMD Customization +# Continuing to use "latest" because this allows a new image to be easily +# created and pushed to the Nexus +ARG DSPACE_VERSION=latest +# End UMD Customization +# The Docker registry to use for DSpace images. Defaults to "docker.io" +# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument +ARG DOCKER_REGISTRY=docker.io + +# Step 1 - Run Maven Build +# UMD Customization +FROM docker.lib.umd.edu/drum-dependencies-8_x:${DSPACE_VERSION} AS build +# End UMD Customization +ARG TARGET_DIR=dspace-installer +WORKDIR /app +# The dspace-installer directory will be written to /install +RUN mkdir /install \ + && chown -Rv dspace: /install \ + && chown -Rv dspace: /app +USER dspace +# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents) +ADD --chown=dspace . /app/ +# Build DSpace +# Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small +# Maven flags here ensure that we skip building test environment and skip all code verification checks. +# These flags speed up this compilation as much as reasonably possible. +ENV MAVEN_FLAGS="-P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true" +RUN mvn --no-transfer-progress package ${MAVEN_FLAGS} && \ + mv /app/dspace/target/${TARGET_DIR}/* /install && \ + mvn clean +# Remove the server webapp to keep image small. +RUN rm -rf /install/webapps/server/ + +# Step 2 - Run Ant Deploy +# UMD Customization +FROM docker.lib.umd.edu/drum-ant:latest AS ant_build +# End UMD Customization +ARG TARGET_DIR=dspace-installer +# COPY the /install directory from 'build' container to /dspace-src in this container +COPY --from=build /install /dspace-src +WORKDIR /dspace-src + +# UMD Customization +# Ant is installed as part of the "drum-ant" Docker image +# End UMD Customization + +# Run necessary 'ant' deploy scripts +RUN ant init_installation update_configs update_code update_webapps + +# Step 3 - Start up DSpace via Runnable JAR +FROM docker.io/eclipse-temurin:${JDK_VERSION} +# NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration. +ENV DSPACE_INSTALL=/dspace +# Copy the /dspace directory from 'ant_build' container to /dspace in this container +COPY --from=ant_build /dspace $DSPACE_INSTALL +WORKDIR $DSPACE_INSTALL +# Need host command for "[dspace]/bin/make-handle-config" +RUN apt-get update \ + && apt-get install -y --no-install-recommends host \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* +# Expose Tomcat port (8080) & Handle Server HTTP port (8000) +EXPOSE 8080 8000 +# Give java extra memory (2GB) +ENV JAVA_OPTS=-Xmx2000m + +# UMD Customization +ENV TZ=America/New_York + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + rsync \ + openssh-client \ + cron \ + csh \ + postfix \ + s-nail \ + libgetopt-complete-perl \ + libconfig-properties-perl \ + vim \ + python3-lxml \ + jq && \ + mkfifo /var/spool/postfix/public/pickup && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +# End UMD Customization +# On startup, run DSpace Runnable JAR +ENTRYPOINT ["java", "-jar", "webapps/server-boot.jar", "--dspace.dir=$DSPACE_INSTALL"] diff --git a/Dockerfile.dev-additions b/Dockerfile.dev-additions new file mode 100644 index 000000000000..4d304ee9357a --- /dev/null +++ b/Dockerfile.dev-additions @@ -0,0 +1,83 @@ +# UMD-provided file running DSpace as part of a "Quick Build" setup +# This file is based on "Dockerfile.dev" + +# This Dockerfile uses JDK17 by default. +# To build with other versions, use "--build-arg JDK_VERSION=[value]" +ARG JDK_VERSION=17 +# The Docker version tag to build from +ARG DSPACE_VERSION=8_x-dev-base +# The Docker registry to use for DSpace images. Defaults to "docker.io" +# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument +ARG DOCKER_REGISTRY=docker.io + +# Step 1 - Run Maven Build +# UMD Customization +FROM docker.lib.umd.edu/drum:${DSPACE_VERSION} AS build +# End UMD Customization +ARG TARGET_DIR=dspace-installer +WORKDIR /app + +USER dspace + +# UMD Customization +# Copy the DSpace source code from the /dspace/modules/[additions]|[server] +# directories from the local machine) into the workdir +# (excluding .dockerignore contents) +RUN rm -rf /app/dspace/modules/additions /app/dspace/modules/server +ADD --chown=dspace dspace/modules/additions /app/dspace/modules/additions +ADD --chown=dspace dspace/modules/server /app/dspace/modules/server +# End UMD Customization + +# Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small +# Maven flags here ensure that we skip building test environment and skip all code verification checks. +# These flags speed up this compilation as much as reasonably possible. +ENV MAVEN_FLAGS="-P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true" +# UMD Customization +RUN mvn package -rf org.dspace:modules -pl '!org.dspace:dspace-iiif,!org.dspace:dspace-oai,!org.dspace:dspace-rdf,!org.dspace:dspace-sword,!org.dspace:dspace-swordv2' ${MAVEN_FLAGS} && \ +# End UMD Customization + mv /app/dspace/target/${TARGET_DIR}/* /install && \ + mvn clean +# Remove the server webapp to keep image small. +RUN rm -rf /install/webapps/server/ + +# Step 2 - Run Ant Deploy +# UMD Customization +FROM docker.lib.umd.edu/drum-ant:latest AS ant_build +# End UMD Customization +ARG TARGET_DIR=dspace-installer +# COPY the /install directory from 'build' container to /dspace-src in this container +COPY --from=build /install /dspace-src +WORKDIR /dspace-src + +# UMD Customization +# Ant is installed as part of the "drum-ant" Docker image +# End UMD Customization + +# Run necessary 'ant' deploy scripts +RUN ant init_installation update_configs update_code update_webapps + +# Step 3 - Start up DSpace via Runnable JAR +FROM docker.io/eclipse-temurin:${JDK_VERSION} +# NOTE: DSPACE_INSTALL must align with the "dspace.dir" default configuration. +ENV DSPACE_INSTALL=/dspace +# Copy the /dspace directory from 'ant_build' container to /dspace in this container +COPY --from=ant_build /dspace $DSPACE_INSTALL +WORKDIR $DSPACE_INSTALL +# Expose Tomcat port +EXPOSE 8080 +# Give java extra memory (2GB) +ENV JAVA_OPTS=-Xmx2000m +# Add csh and Perl libraries for scripts in /dspace/bin +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + csh \ + libgetopt-complete-perl \ + libconfig-properties-perl \ + jq + +# Create the directories needed for Proquest ETD loading +RUN mkdir -p $DSPACE_INSTALL/proquest/incoming $DSPACE_INSTALL/proquest/processed \ + $DSPACE_INSTALL/proquest/csv $DSPACE_INSTALL/proquest/marc + +# On startup, run DSpace Runnable JAR +ENTRYPOINT ["java", "-jar", "webapps/server-boot.jar", "--dspace.dir=$DSPACE_INSTALL"] diff --git a/Dockerfile.dev-base b/Dockerfile.dev-base new file mode 100644 index 000000000000..d31e2c3aaa28 --- /dev/null +++ b/Dockerfile.dev-base @@ -0,0 +1,43 @@ +# UMD-provided file running DSpace as part of a "Quick Build" setup +# This file is based on "Dockerfile.dev" + +# This Dockerfile uses JDK17 by default. +# To build with other versions, use "--build-arg JDK_VERSION=[value]" +ARG JDK_VERSION=17 +# The Docker version tag to build from +# UMD Customization +# Continuing to use "latest" because this allows a new image to be easily +# created and pushed to the Nexus +ARG DSPACE_VERSION=latest +# UMD Customization +# The Docker registry to use for DSpace images. Defaults to "docker.io" +# NOTE: non-DSpace images are hardcoded to use "docker.io" and are not impacted by this build argument +ARG DOCKER_REGISTRY=docker.io + +# Step 1 - Run Maven Build +# UMD Customization +FROM docker.lib.umd.edu/drum-dependencies-8_x:${DSPACE_VERSION} AS build +# End UMD Customization +ARG TARGET_DIR=dspace-installer +WORKDIR /app +# The dspace-installer directory will be written to /install +RUN mkdir /install \ + && chown -Rv dspace: /install \ + && chown -Rv dspace: /app +USER dspace +# Copy the DSpace source code (from local machine) into the workdir (excluding .dockerignore contents) +ADD --chown=dspace . /app/ +# Build DSpace +# Copy the dspace-installer directory to /install. Clean up the build to keep the docker image small +# Maven flags here ensure that we skip building test environment and skip all code verification checks. +# These flags speed up this compilation as much as reasonably possible. +ENV MAVEN_FLAGS="-P-test-environment -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true" +# UMD Customization +RUN mvn install ${MAVEN_FLAGS} && \ + mvn clean +# End UMD Customization + +# Remove the server webapp to keep image small. +RUN rm -rf /install/webapps/server/ + +# UMD Customization - Remaining steps are handled by Dockerfile.dev-additions diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000000..391b99226754 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,146 @@ +pipeline { + // Jenkins configuration dependencies + // + // Global Tool Configuration: + // Git + // + // This configuration utilizes the following Jenkins plugins: + // + // * Warnings Next Generation + // * Email Extension Plugin + // + // This configuration also expects the following environment variables + // to be set (typically in /apps/ci/config/env: + // + // JENKINS_EMAIL_SUBJECT_PREFIX + // The Email subject prefix identifying the server. + // Typically "[Jenkins - ]" where + // is the name of the server, i.e. "[Jenkins - cidev]" + // + // JENKINS_DEFAULT_EMAIL_RECIPIENTS + // A comma-separated list of email addresses that should + // be the default recipients of Jenkins emails. + + agent { + dockerfile { + filename 'Dockerfile.ci' + // Pass JENKINS_EMAIL_SUBJECT_PREFIX and JENKINS_DEFAULT_EMAIL_RECIPIENTS + // into container as "env" arguments, so they are available inside the + // Docker container + args '-u root --env JENKINS_DEFAULT_EMAIL_RECIPIENTS=$JENKINS_DEFAULT_EMAIL_RECIPIENTS --env JENKINS_EMAIL_SUBJECT_PREFIX=$JENKINS_EMAIL_SUBJECT_PREFIX' + } + } + + options { + buildDiscarder( + logRotator( + artifactDaysToKeepStr: '', + artifactNumToKeepStr: '', + numToKeepStr: '20')) + } + + environment { + DEFAULT_RECIPIENTS = "${ \ + sh(returnStdout: true, \ + script: 'echo $JENKINS_DEFAULT_EMAIL_RECIPIENTS').trim() \ + }" + + EMAIL_SUBJECT_PREFIX = "${ \ + sh(returnStdout: true, script: 'echo $JENKINS_EMAIL_SUBJECT_PREFIX').trim() \ + }" + + EMAIL_SUBJECT = "$EMAIL_SUBJECT_PREFIX - " + + '$PROJECT_NAME - ' + + 'GIT_BRANCH_PLACEHOLDER - ' + + '$BUILD_STATUS! - ' + + "Build # $BUILD_NUMBER" + + EMAIL_CONTENT = + '''$PROJECT_NAME - GIT_BRANCH_PLACEHOLDER - $BUILD_STATUS! - Build # $BUILD_NUMBER: + | + |Check console output at $BUILD_URL to view the results. + | + |There are ${ANALYSIS_ISSUES_COUNT} static analysis issues in this build. + | + |There were ${TEST_COUNTS,var="skip"} skipped tests.'''.stripMargin() + } + + stages { + stage('initialize') { + steps { + script { + // Retrieve the actual Git branch being built for use in email. + // + // For pull requests, the actual Git branch will be in the + // CHANGE_BRANCH environment variable. + // + // For actual branch builds, the CHANGE_BRANCH variable won't exist + // (and an exception will be thrown) but the branch name will be + // part of the PROJECT_NAME variable, so it is not needed. + + ACTUAL_GIT_BRANCH = '' + + try { + ACTUAL_GIT_BRANCH = CHANGE_BRANCH + ' - ' + } catch (groovy.lang.MissingPropertyException mpe) { + // Do nothing. A branch (as opposed to a pull request) is being + // built + } + + // Replace the "GIT_BRANCH_PLACEHOLDER" in email variables + EMAIL_SUBJECT = EMAIL_SUBJECT.replaceAll('GIT_BRANCH_PLACEHOLDER - ', ACTUAL_GIT_BRANCH ) + EMAIL_CONTENT = EMAIL_CONTENT.replaceAll('GIT_BRANCH_PLACEHOLDER - ', ACTUAL_GIT_BRANCH ) + } + } + } + + // Only running "test" because DSpace runs the unit/integration tests as + // part of the build. + stage('test') { + steps { + sh ''' + mvn clean install -DskipUnitTests=false -DskipIntegrationTests=false -Dsurefire.rerunFailingTestsCount=3 -Dfailsafe.rerunFailingTestsCount=3 + ''' + } + post { + always { + // Collect the unit test reports + junit '**/target/surefire-reports/*.xml' + + // Collect integration test reports + junit '**/target/failsafe-reports/*.xml' + + recordIssues( + tools: [checkStyle(reportEncoding: 'UTF-8')], + // Filter out the hundreds of TODOs warnings at the "INFO" level + // from the stock DSpace code + filters: [excludeType('TodoCommentCheck')], + qualityGates: [[threshold: 1, type: 'TOTAL', criticality: 'UNSTABLE']], + enabledForFailure: true) + } + } + } + + stage('clean-workspace') { + steps { + // Change permissions of the workspace directory to world-writeable + // so Jenkins can delete it. This is needed, because files may be + // written to the directory from the Docker container as the "root" + // user, which Jenkins would not otherwise be able to clean up. + sh ''' + chmod --recursive 777 $WORKSPACE + ''' + + cleanWs() + } + } + } + + post { + always { + emailext to: "$DEFAULT_RECIPIENTS", + subject: "$EMAIL_SUBJECT", + body: "$EMAIL_CONTENT" + } + } +} diff --git a/README-DRUM.md b/README-DRUM.md new file mode 100644 index 000000000000..676af9d9dbda --- /dev/null +++ b/README-DRUM.md @@ -0,0 +1,151 @@ +# Digital Repository at the University of Maryland (DRUM) + +Home: + +## Documentation + +The original DSpace documentation is in the "README.md" file. + +## Development Environment + +Instructions for building and running drum locally can be found in +[dspace/docs/DockerDevelopmentEnvironment.md](/dspace/docs/DockerDevelopmentEnvironment.md) + +## Building Images for K8s Deployment + +As of May 2023, MacBooks utilizing Apple Silicon (the "arm64" architecture) +are unable to directly generate the "amd64" Docker images used by Kubernetes. + +The following procedure uses the Docker "buildx" functionality and the +Kubernetes "build" namespace to build the Docker images. This procedure should +work on both "arm64" and "amd64" MacBooks. + +All images will be automatically pushed to the Nexus. + +### Local Machine Setup + +See in +Confluence for information about setting up a MacBook to use the Kubernetes +"build" namespace. + +### Creating the Docker images + +1) In an empty directory, checkout the Git repository and switch into the + directory: + + ```bash + $ git clone git@github.com:umd-lib/DSpace.git drum + $ cd drum + ``` + +2) Checkout the appropriate Git tag, branch, or commit for the Docker images. + +3) Set up a "DRUM_TAG" environment variable: + + ```bash + $ export DRUM_TAG= + ``` + + where \ is the Docker image tag to associate with the + Docker images. This will typically be the Git tag for the DRUM version, + or some other identifier, such as a Git commit hash. For example, using the + Git tag of "7.4-drum-0": + + ```bash + $ export DRUM_TAG=7.4-drum-0 + ``` + +4) Set up a "DRUM_DIR" environment variable referring to the current + directory: + + ```bash + $ export DRUM_DIR=`pwd` + ``` + +5) Switch to the Kubernetes "build" namespace: + + ```bash + $ kubectl config use-context build + ``` + +6) Create the "docker.lib.umd.edu/drum-dependencies-8_x" Docker image. This + image is used to pre-cache Maven downloads that will be used in subsequent + DSpace docker builds: + + ```bash + $ docker buildx build --platform linux/amd64 --builder=kube --push --no-cache -t docker.lib.umd.edu/drum-dependencies-8_x:latest -f Dockerfile.dependencies . + ``` + +7) Create the "docker.lib.umd.edu/drum" Docker image: + + ```bash + $ docker buildx build --platform linux/amd64 --builder=kube --push --no-cache -f Dockerfile -t docker.lib.umd.edu/drum:$DRUM_TAG . + ``` + +8) Create the "docker.lib.umd.edu/dspace-postgres", which is a Postgres image + with "pgcrypto" module: + + **Note:** The "Dockerfile" for the "dspace-postgres" image specifies + only the major Postgres version as the base image. This allows Postgres + minor version updates to be retrieved automatically. It may not be + necessary to create new "dspace-postgres" image versions for every DRUM + patch or hotfix version increment. + + ```bash + $ cd $DRUM_DIR/dspace/src/main/docker/dspace-postgres-pgcrypto + + $ docker buildx build --platform linux/amd64 --builder=kube --push --no-cache -f Dockerfile -t docker.lib.umd.edu/dspace-postgres:$DRUM_TAG . + ``` + +9) Create the "docker.lib.umd.edu/drum-solr": + + **Note:** The "Dockerfile" for the "drum-solr" image specifies only the + major Solr version as the base image. This allows Solr minor version updates + to be retrieved automatically. It may not be necessary to create new + "drum-solr" image versions for every DRUM patch or hotfix version increment. + + ```bash + $ cd $DRUM_DIR/dspace/solr + + $ docker buildx build --platform linux/amd64 --builder=kube --push --no-cache -f Dockerfile -t docker.lib.umd.edu/drum-solr:$DRUM_TAG . + ``` + +### Features + +* [DrumFeatures](dspace/docs/DrumFeatures.md) - Summary of DRUM enhancements to + base DSpace functionality +* [DrumConfigurationCustomization](dspace/docs/DrumConfigurationCustomization.md) - + Information about customizing DSpace for DRUM. +* [docs](dspace/docs) - additional documentation + +## Customization Markings + +UMD customizations to stock DSpace code should be marked, if possible, with +a starting comment "UMD Customization" and an ending comment of +"End UMD Customization", for example, in a Java file: + +```java +// UMD Customization +... New or modified code ... +// End UMD Customization +``` + +The following customizations *do not* need to be commented: + +* Updates to the "\" identifier in "pom.xml" files +* "Branding" changes in email templates such as "dspace/config/emails/" or + the default DSpace license in "dspace/config/default.license", as these files + do not have a convenient "comment" mechanism +* Files that do not have a "comment" mechanism, such as JSON files +* Extremely trivial whitespace changes unrelated to UMD customizations, such as + tabs in the modified DSpace file being automatically converted to spaces by + VS Code, or an end-of-file line. + +The main goal is to make it immediately when performing DSpace version upgrades +whether a change in a file is due to an explicit UMD customization. + +## License + +See the [DRUM-LICENSE](DRUM-LICENSE.md) file for license rights and limitations +(Apache 2.0). This license only governs the part of code base developed at UMD. +The DSpace license can be found at diff --git a/docker-compose-cli.yml b/docker-compose-cli.yml index 5d15845fa8df..942354f4d5c8 100644 --- a/docker-compose-cli.yml +++ b/docker-compose-cli.yml @@ -6,7 +6,9 @@ networks: external: true services: dspace-cli: - image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-8_x}" + # UMD Customization + image: "docker.lib.umd.edu/drum-cli:${DSPACE_VER:-dspace-8_x}" + # End UMD Customization container_name: dspace-cli build: context: . @@ -19,7 +21,9 @@ services: # dspace.dir: Must match with Dockerfile's DSPACE_INSTALL directory. dspace__P__dir: /dspace # db.url: Ensure we are using the 'dspacedb' image for our database - db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace' + # UMD Customization + db__P__url: 'jdbc:postgresql://dspacedb:5432/drum' + # End UMD Customization # solr.server: Ensure we are using the 'dspacesolr' image for Solr solr__P__server: http://dspacesolr:8983/solr volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 9177ff4bd977..36ff1cf6f63b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,21 @@ networks: # If you customize this value, be sure to customize the 'proxies.trusted.ipranges' env variable below. - subnet: 172.23.0.0/16 services: + # UMD Customization + # Nginx server configuration for supporting HTTPS connections from the + # local development environment + nginx: + container_name: nginx + image: nginx:1.25.2 + ports: + - "80:80" + - "443:8443" + volumes: + - ./dspace/src/main/docker/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./dspace/src/main/docker/nginx/certs:/etc/nginx/certs:ro + networks: + dspacenet: + # End UMD Customization # DSpace (backend) webapp container dspace: container_name: dspace @@ -17,21 +32,34 @@ services: # dspace.dir: Must match with Dockerfile's DSPACE_INSTALL directory. dspace__P__dir: /dspace # Uncomment to set a non-default value for dspace.server.url or dspace.ui.url - # dspace__P__server__P__url: http://localhost:8080/server - # dspace__P__ui__P__url: http://localhost:4000 + # UMD Customization + dspace__P__server__P__url: https://api.drum-local.lib.umd.edu/server + dspace__P__ui__P__url: https://drum-local.lib.umd.edu:4000 + # End UMD Customization dspace__P__name: 'DSpace Started with Docker Compose' # db.url: Ensure we are using the 'dspacedb' image for our database - db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace' + # UMD Customization + db__P__url: 'jdbc:postgresql://dspacedb:5432/drum' + # End UMD Customization # solr.server: Ensure we are using the 'dspacesolr' image for Solr solr__P__server: http://dspacesolr:8983/solr + # matomo.tracker.url: Ensure we are using the 'matomo' image for Matomo + matomo__P__tracker__P__url: http://matomo # proxies.trusted.ipranges: This setting is required for a REST API running in Docker to trust requests # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above. proxies__P__trusted__P__ipranges: '172.23.0' LOGGING_CONFIG: /dspace/config/log4j2-container.xml - image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-8_x-test}" + # UMD Customization + JPDA_OPTS: "-agentlib:jdwp=transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n" + # End UMD Customization + # UMD Customization + image: "docker.lib.umd.edu/drum:${DSPACE_VER:-8_x-dev}" + # End UMD Customization build: context: . - dockerfile: Dockerfile.test + # UMD Customization + dockerfile: Dockerfile.dev + # End UMD Customization depends_on: - dspacedb networks: @@ -59,18 +87,26 @@ services: - | while (! /dev/null 2>&1; do sleep 1; done; /dspace/bin/dspace database migrate - java -jar /dspace/webapps/server-boot.jar --dspace.dir=/dspace + # UMD Customization + java $${JPDA_OPTS} -jar /dspace/webapps/server-boot.jar --dspace.dir=/dspace + # End UMD Customization # DSpace PostgreSQL database container dspacedb: container_name: dspacedb # Uses a custom Postgres image with pgcrypto installed - image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}" + # UMD Customization + image: docker.lib.umd.edu/dspace-postgres:${DSPACE_VER:-latest} + # End UMD Customization build: # Must build out of subdirectory to have access to install script for pgcrypto context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ environment: PGDATA: /pgdata - POSTGRES_PASSWORD: dspace + # UMD Customization + POSTGRES_DB: drum + POSTGRES_USER: drum + POSTGRES_PASSWORD: drum + # End UMD Customization networks: dspacenet: ports: @@ -81,6 +117,9 @@ services: volumes: # Keep Postgres data directory between reboots - pgdata:/pgdata + # UMD Customization + - ./postgres-init:/docker-entrypoint-initdb.d + # End UMD Customization # DSpace Solr container dspacesolr: container_name: dspacesolr diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 487d5cab953f..ecfdd6542ffa 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -12,7 +12,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT .. @@ -349,6 +349,13 @@ org.apache.logging.log4j log4j-slf4j2-impl + + + org.apache.logging.log4j + log4j-layout-template-json + runtime + + org.hibernate.orm hibernate-core @@ -771,7 +778,6 @@ 1.19.0 test - com.opencsv @@ -813,7 +819,7 @@ - + eu.openaire broker-client @@ -847,7 +853,7 @@ - + io.findify s3mock_2.13 diff --git a/dspace-api/src/main/java/org/dspace/checker/CheckerCommand.java b/dspace-api/src/main/java/org/dspace/checker/CheckerCommand.java index a12ac3b98a2e..c82eb365277d 100644 --- a/dspace-api/src/main/java/org/dspace/checker/CheckerCommand.java +++ b/dspace-api/src/main/java/org/dspace/checker/CheckerCommand.java @@ -131,7 +131,13 @@ public void process() throws SQLException { collector.collect(context, info); } - context.uncacheEntity(bitstream); + // UMD Customization + // This change was provided to DSpace in Pull Request 10508 + // This customization markers can be removed once the + // application has been upgraded to a DSpace version containing + // the pull request. + context.commit(); + // End UMD Customization bitstream = dispatcher.next(); } } diff --git a/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java index bd56ad465163..880a72d0a6c7 100644 --- a/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/BitstreamServiceImpl.java @@ -12,8 +12,10 @@ import java.sql.SQLException; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.regex.Pattern; +import java.util.stream.Collectors; import jakarta.annotation.Nullable; import org.apache.commons.collections4.CollectionUtils; @@ -496,4 +498,14 @@ public List getNotReferencedBitstreams(Context context) throws SQLExc public Long getLastModified(Bitstream bitstream) throws IOException { return bitstreamStorageService.getLastModified(bitstream); } + + @Override + public boolean isInBundle(Bitstream bitstream, java.util.Collection bundleNames) throws SQLException { + Set bundles = + bitstream.getBundles() + .stream() + .map(Bundle::getName) + .collect(Collectors.toSet()); + return bundleNames.stream().anyMatch(bundles::contains); + } } diff --git a/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java b/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java index c22428f11a96..2af76166492b 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/BitstreamService.java @@ -235,4 +235,14 @@ public InputStream retrieve(Context context, Bitstream bitstream) */ @Nullable Long getLastModified(Bitstream bitstream) throws IOException; + + /** + * Checks if the given bitstream is inside one of the bundle + * + * @param bitstream bitstream to verify + * @param bundleNames names of the bundles to serch for + * @return true if is in one of the bundles, false otherwise + * @throws SQLException + */ + boolean isInBundle(Bitstream bitstream, java.util.Collection bundleNames) throws SQLException; } diff --git a/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java b/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java index 68c492d1a9a0..d628478e1b21 100644 --- a/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java +++ b/dspace-api/src/main/java/org/dspace/google/GoogleAsyncEventListener.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -23,8 +24,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.content.Bitstream; -import org.dspace.content.Bundle; import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamService; import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.google.client.GoogleAnalyticsClient; @@ -57,6 +58,9 @@ public class GoogleAsyncEventListener extends AbstractUsageEventListener { @Autowired private ClientInfoService clientInfoService; + @Autowired + private BitstreamService bitstreamService; + @Autowired private List googleAnalyticsClients; @@ -181,25 +185,35 @@ private String getDocumentPath(HttpServletRequest request) { */ private boolean isContentBitstream(UsageEvent usageEvent) { // check if event is a VIEW event and object is a Bitstream - if (usageEvent.getAction() == UsageEvent.Action.VIEW - && usageEvent.getObject().getType() == Constants.BITSTREAM) { - // check if bitstream belongs to a configured bundle - List allowedBundles = List.of(configurationService - .getArrayProperty("google-analytics.bundles", new String[]{Constants.CONTENT_BUNDLE_NAME})); - if (allowedBundles.contains("none")) { - // GA events for bitstream views were turned off in config - return false; - } - List bitstreamBundles; - try { - bitstreamBundles = ((Bitstream) usageEvent.getObject()) - .getBundles().stream().map(Bundle::getName).collect(Collectors.toList()); - } catch (SQLException e) { - throw new RuntimeException(e.getMessage(), e); - } - return allowedBundles.stream().anyMatch(bitstreamBundles::contains); + if (!isBitstreamView(usageEvent)) { + return false; + } + // check if bitstream belongs to a configured bundle + Set allowedBundles = + Set.of( + configurationService.getArrayProperty( + "google-analytics.bundles", + new String[]{Constants.CONTENT_BUNDLE_NAME} + ) + ); + if (allowedBundles.contains("none")) { + // GA events for bitstream views were turned off in config + return false; + } + return isInBundle((Bitstream) usageEvent.getObject(), allowedBundles); + } + + private boolean isInBundle(Bitstream bitstream, Set allowedBundles) { + try { + return this.bitstreamService.isInBundle(bitstream, allowedBundles); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); } - return false; + } + + private boolean isBitstreamView(UsageEvent usageEvent) { + return usageEvent.getAction() == UsageEvent.Action.VIEW + && usageEvent.getObject().getType() == Constants.BITSTREAM; } private boolean isGoogleAnalyticsKeyNotConfigured() { diff --git a/dspace-api/src/main/java/org/dspace/matomo/MatomoAbstractHandler.java b/dspace-api/src/main/java/org/dspace/matomo/MatomoAbstractHandler.java new file mode 100644 index 000000000000..7ce8a7ff5b59 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/MatomoAbstractHandler.java @@ -0,0 +1,43 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import org.dspace.matomo.client.MatomoClient; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.matomo.model.MatomoRequestDetailsBuilder; +import org.dspace.usage.UsageEvent; + +/** + * This class represents an abstract class that will be used to handle the {@code UsageEvent}. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public abstract class MatomoAbstractHandler implements MatomoUsageEventHandler { + protected final MatomoClient matomoClient; + protected final MatomoRequestDetailsBuilder builder; + + public MatomoAbstractHandler(MatomoClient matomoClient, MatomoRequestDetailsBuilder builder) { + this.matomoClient = matomoClient; + this.builder = builder; + } + + /** + * Converts a DSpace usage event into a Matomo request details object. + * + * @param usageEvent The DSpace usage event to convert + * @return MatomoRequestDetails object containing the converted event details + */ + protected MatomoRequestDetails toDetails(UsageEvent usageEvent) { + return this.builder.build(usageEvent); + } + @Override + public abstract void handleEvent(UsageEvent usageEvent); + + @Override + public abstract void sendEvents(); +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/MatomoAsyncBulkRequestHandler.java b/dspace-api/src/main/java/org/dspace/matomo/MatomoAsyncBulkRequestHandler.java new file mode 100644 index 000000000000..70bd8115528b --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/MatomoAsyncBulkRequestHandler.java @@ -0,0 +1,79 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import java.util.ArrayList; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.matomo.client.MatomoClient; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.matomo.model.MatomoRequestDetailsBuilder; +import org.dspace.matomo.model.MatomoRequestDetailsSplitter; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * This class groups together {@code capacity} requests that will be sent as one bulk request + * using the {@code MatomoClient} + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoAsyncBulkRequestHandler extends MatomoAbstractHandler { + + private static final Logger log = LogManager.getLogger(MatomoAsyncBulkRequestHandler.class); + + private final LinkedBlockingDeque deque; + private final Lock lock = new ReentrantLock(); + + + public MatomoAsyncBulkRequestHandler( + @Autowired MatomoRequestDetailsBuilder builder, + @Autowired MatomoClient matomoClient, + int capacity + ) { + super(matomoClient, builder); + this.deque = new LinkedBlockingDeque<>(capacity); + } + + @Override + public void handleEvent(UsageEvent usageEvent) { + if (usageEvent == null) { + log.error("Skipping UsageEvent is null"); + return; + } + + lock.lock(); + try { + this.deque.add(this.toDetails(usageEvent)); + if (this.deque.remainingCapacity() <= 1) { + this.sendEvents(); + } + } finally { + lock.unlock(); + } + } + + @Override + public void sendEvents() { + lock.lock(); + try { + ArrayList details = new ArrayList<>(); + deque.drainTo(details); + MatomoRequestDetailsSplitter.split(details) + .values() + .forEach(this.matomoClient::sendDetails); + } finally { + lock.unlock(); + } + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/MatomoEventListener.java b/dspace-api/src/main/java/org/dspace/matomo/MatomoEventListener.java new file mode 100644 index 000000000000..1a4f66e4567d --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/MatomoEventListener.java @@ -0,0 +1,123 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import java.sql.SQLException; +import java.util.List; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.content.Bitstream; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Constants; +import org.dspace.services.ConfigurationService; +import org.dspace.services.model.Event; +import org.dspace.usage.AbstractUsageEventListener; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * This EventListener handles {@code UsageEvent}s and send them to all the {@code List} + * configured + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoEventListener extends AbstractUsageEventListener { + + private static final Logger log = LogManager.getLogger(MatomoEventListener.class); + + private final ConfigurationService configurationService; + private final BitstreamService bitstreamService; + private final List matomoUsageEventHandlers; + + public MatomoEventListener( + @Autowired List matomoUsageEventHandlers, + @Autowired ConfigurationService configurationService, + @Autowired BitstreamService bitstreamService + ) { + this.matomoUsageEventHandlers = matomoUsageEventHandlers; + this.configurationService = configurationService; + this.bitstreamService = bitstreamService; + } + + @Override + public void receiveEvent(Event event) { + if (!(event instanceof UsageEvent usageEvent)) { + return; + } + + try { + if (!matomoEnabled() || matomoUsageEventHandlers == null || matomoUsageEventHandlers.isEmpty()) { + return; + } + + if (!isContentBitstream(usageEvent)) { + return; + } + + if (log.isDebugEnabled()) { + log.debug("Usage event received {}", event.getName()); + } + + this.matomoUsageEventHandlers.forEach(handler -> handler.handleEvent(usageEvent)); + + } catch (Exception e) { + log.error("Failed to add the UsageEvent to Matomo Handlers {} ", usageEvent, e); + } + } + + private boolean matomoEnabled() { + return this.configurationService.getBooleanProperty("matomo.enabled", false); + } + + /** + * Verifies if the usage event is a content bitstream view event, by checking if: + *
    + *
  • the usage event is a view event
    • + *
  • the object of the usage event is a bitstream
    • + *
  • the bitstream belongs to one of the configured bundles (fallback: ORIGINAL bundle)
    • + *
+ */ + private boolean isContentBitstream(UsageEvent usageEvent) { + // check if event is a VIEW event and object is a Bitstream + if (!isBitstreamView(usageEvent)) { + return false; + } + // check if bitstream belongs to a configured bundle + Set allowedBundles = getTrackedBundles(); + if (allowedBundles.contains("none")) { + // events for bitstream views were turned off in config + return false; + } + return isInBundle(((Bitstream) usageEvent.getObject()), allowedBundles); + } + + private Set getTrackedBundles() { + return Set.of( + configurationService.getArrayProperty( + "matomo.track.bundles", + new String[] {Constants.CONTENT_BUNDLE_NAME} + ) + ); + } + + protected boolean isInBundle(Bitstream bitstream, Set allowedBundles) { + try { + return this.bitstreamService.isInBundle(bitstream, allowedBundles); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + private boolean isBitstreamView(UsageEvent usageEvent) { + return usageEvent.getAction() == UsageEvent.Action.VIEW + && usageEvent.getObject().getType() == Constants.BITSTREAM; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/MatomoSyncEventHandler.java b/dspace-api/src/main/java/org/dspace/matomo/MatomoSyncEventHandler.java new file mode 100644 index 000000000000..e18b7f9d9b05 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/MatomoSyncEventHandler.java @@ -0,0 +1,46 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.matomo.client.MatomoClient; +import org.dspace.matomo.model.MatomoRequestDetailsBuilder; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * This class represents a sync event handler that will send details one by one using the {@code MatomoClient} + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoSyncEventHandler extends MatomoAbstractHandler { + + private static final Logger log = LogManager.getLogger(MatomoSyncEventHandler.class); + + public MatomoSyncEventHandler( + @Autowired MatomoClient matomoClient, + @Autowired MatomoRequestDetailsBuilder builder + ) { + super(matomoClient, builder); + } + + @Override + public void handleEvent(UsageEvent usageEvent) { + if (usageEvent == null) { + log.error("UsageEvent is null"); + return; + } + matomoClient.sendDetails(toDetails(usageEvent)); + } + + @Override + public void sendEvents() { + log.warn("Events are sent synchronously, you don't need to use this method!"); + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/MatomoUsageEventHandler.java b/dspace-api/src/main/java/org/dspace/matomo/MatomoUsageEventHandler.java new file mode 100644 index 000000000000..2c34947f9e81 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/MatomoUsageEventHandler.java @@ -0,0 +1,30 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import org.dspace.usage.UsageEvent; + +/** + * This interface represents an Handler to track {@code UsageEvent} towards Matomo. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public interface MatomoUsageEventHandler { + /** + * This method handles the {@code UsageEvent} in order to send that request + * to Matomo + * + * @param usageEvent + */ + void handleEvent(UsageEvent usageEvent); + + /** + * Sends all the pending request to the Matomo tracking endpoint. + */ + void sendEvents(); +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoAbstractClient.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoAbstractClient.java new file mode 100644 index 000000000000..f69192e173b4 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoAbstractClient.java @@ -0,0 +1,169 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import java.net.HttpURLConnection; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.matomo.exception.MatomoClientException; +import org.dspace.matomo.model.MatomoCookieConverter; +import org.dspace.matomo.model.MatomoRequestDetails; + +/** + * + * {@code abstract} client for Matomo integration + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public abstract class MatomoAbstractClient implements MatomoClient { + + protected final String baseUrl; + protected final String token; + protected final MatomoRequestBuilder matomoRequestBuilder; + protected final MatomoResponseReader matomoResponseReader; + protected final C httpClient; + protected final Logger log = LogManager.getLogger(getClass()); + + public MatomoAbstractClient( + String baseUrl, String token, + MatomoRequestBuilder matomoRequestBuilder, MatomoResponseReader matomoResponseReader, + C httpClient + ) { + this.baseUrl = baseUrl; + this.token = token; + this.matomoRequestBuilder = matomoRequestBuilder; + this.matomoResponseReader = matomoResponseReader; + this.httpClient = httpClient; + } + + /** + * Creates a request with the given request body and empty cookies. + * + * @param requestBody The body content of the request + * @return A request object of type T + */ + protected T createRequest(String requestBody) { + return createRequest(requestBody, ""); + } + + protected abstract T createRequest(String requestBody, String cookies); + + protected abstract void executeRequest(String requestBody, String cookies, BiConsumer responseConsumer); + + protected abstract int getStatusCode(U response); + + protected abstract String getResponseContent(U response); + + @Override + public void sendDetails(MatomoRequestDetails... details) { + this.sendDetails(Arrays.asList(details)); + } + + protected String createRequestBody(List details) { + return this.matomoRequestBuilder.buildJSON(new MatomoBulkRequest(token, details)); + } + + public void sendDetails(List details) { + if (details == null || details.isEmpty()) { + log.warn("Cannot send an empty request!"); + return; + } + + if (StringUtils.isEmpty(baseUrl)) { + log.error("Cannot send these details {} to Matomo - No endpoint configured!", details); + return; + } + + try { + this.executeRequest( + createRequestBody(details), + generateCookies(details), + this::logError + ); + } catch (Exception ex) { + throw new MatomoClientException("An error occurs sending events to " + baseUrl, ex); + } + } + + /** + * Generates a cookie string from a list of Matomo request details. + * + * @param details List of MatomoRequestDetails to extract cookie information from + * @return String containing the formatted cookie data + */ + protected String generateCookies(List details) { + return MatomoCookieConverter.convert(details); + } + + /** + * Adds cookies to an HTTP connection by setting the Cookie request property. + * Takes a map of cookie names and values and formats them into a single cookie header string. + * + * @param connection The HttpURLConnection to add cookies to + * @param cookies Map containing cookie names as keys and cookie values as values + */ + static void addCookies( + HttpURLConnection connection, Map cookies + ) { + StringBuilder cookiesValue = new StringBuilder(); + if (cookies != null) { + for (Iterator> iterator = cookies.entrySet().iterator(); iterator.hasNext(); ) { + Map.Entry entry = iterator.next(); + cookiesValue.append(entry.getKey()).append("=").append(entry.getValue()); + if (iterator.hasNext()) { + cookiesValue.append("; "); + } + } + String requestCookies = connection.getRequestProperty("Cookie"); + if (!StringUtils.isEmpty(requestCookies)) { + cookiesValue.append("; ").append(requestCookies); + } + } + if (cookiesValue.length() > 0) { + connection.setRequestProperty("Cookie", cookiesValue.toString()); + } + } + + protected void logError(U response, String requestBody) { + if (isNotSuccessful(response)) { + String responseMessage = formatErrorMessage(response); + log.error( + "Cannot register the event on Matomo, REQUEST: {} - RESPONSE: {}", + requestBody, + responseMessage + ); + throw new MatomoClientException(responseMessage); + } + String responseBody = getResponseContent(response); + MatomoResponse matomoResponse = matomoResponseReader.fromJSON(responseBody); + if ( + matomoResponse == null || + !MatomoResponse.SUCCESS.equals(matomoResponse.status()) || + matomoResponse.invalid() > 0 + ) { + log.error("Unable to track requestBody: {}, response was: {}", requestBody, responseBody); + } + } + + protected boolean isNotSuccessful(U response) { + int statusCode = getStatusCode(response); + return statusCode < 200 || statusCode > 299; + } + + protected String formatErrorMessage(U response) { + return "Status " + getStatusCode(response) + ". Content: " + getResponseContent(response); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoAsyncClientImpl.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoAsyncClientImpl.java new file mode 100644 index 000000000000..7a1e39f20234 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoAsyncClientImpl.java @@ -0,0 +1,89 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import java.net.ProxySelector; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.function.BiConsumer; + +/** + * + * {@code MatomoAbstractClient} implementation that handles communication with the Matomo service + * by using async methods with {@code CompletableFuture}. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoAsyncClientImpl extends MatomoAbstractClient> { + + public MatomoAsyncClientImpl( + String baseUrl, String token, + MatomoRequestBuilder matomoRequestBuilder, + MatomoResponseReader matomoResponseReader + ) { + this( + baseUrl, token, matomoRequestBuilder, matomoResponseReader, + HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(Duration.ofSeconds(5)) + .proxy(ProxySelector.getDefault()) + .build() + ); + } + + public MatomoAsyncClientImpl( + String baseUrl, String token, + MatomoRequestBuilder matomoRequestBuilder, + MatomoResponseReader matomoResponseReader, + HttpClient httpClient) { + super(baseUrl, token, matomoRequestBuilder, matomoResponseReader, httpClient); + } + + + @Override + protected HttpRequest createRequest(String requestBody, String cookies) { + return HttpRequest.newBuilder(URI.create(baseUrl)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .setHeader("Content-Type", "application/json") + .setHeader("Cookie", cookies) + .build(); + } + + @Override + protected void executeRequest( + String requestBody, + String cookies, + BiConsumer, String> responseConsumer + ) { + httpClient + .sendAsync(createRequest(requestBody, cookies), java.net.http.HttpResponse.BodyHandlers.ofString()) + .thenAccept(response -> responseConsumer.accept(response, requestBody)) + .exceptionally(this::logError); + } + + private Void logError(Throwable throwable) { + log.error("Cannot track this request to Matomo! Check the matomo.tracking.url configured. ", throwable); + return null; + } + + protected int getStatusCode(HttpResponse response) { + return response.statusCode(); + } + + protected String getResponseContent(HttpResponse response) { + try { + return response.body(); + } catch (Exception e) { + log.error("An error occurs getting the response content", e); + return "Generic error"; + } + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoBulkRequest.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoBulkRequest.java new file mode 100644 index 000000000000..eb95ed38efcf --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoBulkRequest.java @@ -0,0 +1,26 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.matomo.model.MatomoRequestDetailsListConverter; + +/** + * Request that will be sent to Matomo tracking endpoint + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +record MatomoBulkRequest( + @JsonProperty(value = "token_auth", required = true) String token, + @JsonSerialize(using = MatomoRequestDetailsListConverter.class) List requests) { + +} \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoClient.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoClient.java new file mode 100644 index 000000000000..e2619ee2afd3 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoClient.java @@ -0,0 +1,33 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import java.util.List; + +import org.dspace.matomo.model.MatomoRequestDetails; + +/** + * This interface can be used to implement a HTTP client that will connect + * to the Matomo Tracking api. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public interface MatomoClient { + + /** + * This method sends tracked resources to Matomo Tracking API + * @param details + */ + void sendDetails(MatomoRequestDetails... details); + + /** + * This method is an overload of the {@code sendDetails} above + * @param details + */ + void sendDetails(List details); +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoClientImpl.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoClientImpl.java new file mode 100644 index 000000000000..c610ec571e65 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoClientImpl.java @@ -0,0 +1,80 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.function.BiConsumer; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.dspace.matomo.exception.MatomoClientException; + +/** + * Simple synchronous client for Matomo that uses an {@code CloseableHttpClient} to send out + * requests. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoClientImpl extends MatomoAbstractClient { + + public MatomoClientImpl( + String baseUrl, String token, + MatomoRequestBuilder matomoRequestBuilder, + MatomoResponseReader matomoResponseReader, + CloseableHttpClient httpClient + ) { + super(baseUrl, token, matomoRequestBuilder, matomoResponseReader, httpClient); + } + + @Override + protected void executeRequest( + String requestBody, String cookies, BiConsumer responseConsumer + ) { + try (CloseableHttpResponse response = httpClient.execute(createRequest(requestBody, cookies))) { + responseConsumer.accept(response, requestBody); + } catch (MatomoClientException ex) { + throw ex; + } catch (Exception ex) { + throw new MatomoClientException("An error occurs sending events to " + baseUrl, ex); + } + } + + @Override + protected HttpPost createRequest(String requestBody, String cookies) { + HttpPost httpPost = new HttpPost(baseUrl); + try { + httpPost.setHeader("Cookie", cookies); + httpPost.setEntity(new StringEntity(requestBody)); + } catch (UnsupportedEncodingException e) { + throw new MatomoClientException("Error creating request", e); + } + return httpPost; + } + + + @Override + protected int getStatusCode(HttpResponse response) { + return response.getStatusLine().getStatusCode(); + } + + @Override + protected String getResponseContent(HttpResponse response) { + try { + return IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); + } catch (UnsupportedOperationException | IOException e) { + log.error("An error occurs getting the response content", e); + return "Generic error"; + } + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoRequestBuilder.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoRequestBuilder.java new file mode 100644 index 000000000000..46a3a553022a --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoRequestBuilder.java @@ -0,0 +1,48 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * This class is a {@code JSONSerializer} that will convert a {@code MatomoBulkRequest} into a proper JSON + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestBuilder { + + private static final Logger log = LogManager.getLogger(MatomoRequestBuilder.class); + ObjectMapper objectMapper; + + { + objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + } + + /** + * This method converts a {@code MatomoBulkRequest} request into a JSON + * @param request a {@code MatomoBulkRequest} object + * @return String + */ + String buildJSON(MatomoBulkRequest request) { + if (request == null) { + return null; + } + try { + return objectMapper.writeValueAsString(request); + } catch (JsonProcessingException e) { + log.error("Cannot convert the Matomo request properly!", e); + } + return null; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoResponse.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoResponse.java new file mode 100644 index 000000000000..22be013bd281 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoResponse.java @@ -0,0 +1,21 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * This record represents the response of the Matomo Tracking API. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +record MatomoResponse(String status, int tracked, int invalid, @JsonProperty("invalid_indices") int[] invalidIndices) { + + public static final String SUCCESS = "success"; + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/client/MatomoResponseReader.java b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoResponseReader.java new file mode 100644 index 000000000000..9dcad3c17ac0 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/client/MatomoResponseReader.java @@ -0,0 +1,48 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * This class represents a custom {@code JSONDeserializer} that converts the JSON into {@code MatomoResponse}. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoResponseReader { + + private static final Logger log = LogManager.getLogger(MatomoResponseReader.class); + ObjectMapper objectMapper; + + { + objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + } + + /** + * Converts a String response into a {@code MatomoResponse} object + * @param response + * @return + */ + MatomoResponse fromJSON(String response) { + if (response == null) { + return null; + } + try { + return objectMapper.readValue(response, MatomoResponse.class); + } catch (JsonProcessingException e) { + log.error("Cannot convert the Matomo response: {} properly!", response, e); + } + return null; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/exception/MatomoClientException.java b/dspace-api/src/main/java/org/dspace/matomo/exception/MatomoClientException.java new file mode 100644 index 000000000000..9319d50fba5f --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/exception/MatomoClientException.java @@ -0,0 +1,28 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.exception; + +/** + * This class represents an Exception that will be used to encapsulate details coming from {@code Matomo}. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoClientException extends RuntimeException { + + public MatomoClientException(String message, Throwable cause) { + super(message, cause); + } + + public MatomoClientException(String message) { + super(message); + } + + public MatomoClientException(Throwable cause) { + super(cause); + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCookieIdentifierEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCookieIdentifierEnricher.java new file mode 100644 index 000000000000..bf791d5a54cc --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCookieIdentifierEnricher.java @@ -0,0 +1,57 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import jakarta.servlet.http.Cookie; +import org.apache.commons.lang3.StringUtils; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * Enricher that extracts any {@code _pk_id} cookie sent inside the request + * to track the same {@code id} used to track interaction on the angular side.
+ * The cookie will have a similar format: {@code _pk_id.1.1fff=3225aebdb98b13f9.1740076196.}
+ * Where only the first 16 hexadecimal characters represents the id for Matomo. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestCookieIdentifierEnricher implements MatomoRequestDetailsEnricher { + + static final String _PK_ID_NAME = "_pk_id"; + static final Pattern _pk_id = Pattern.compile("(^([a-f]|[0-9]){16})"); + + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + Cookie[] cookies = usageEvent.getRequest().getCookies(); + if (cookies == null || cookies.length == 0) { + return matomoRequestDetails; + } + return getCookie(cookies).filter(StringUtils::isNotEmpty) + .map(id -> matomoRequestDetails.addParameter("_id", id)) + .orElse(matomoRequestDetails); + } + + public static boolean hasCookie(Cookie[] cookies) { + return cookies != null && getCookie(cookies).isPresent(); + } + + public static Optional getCookie(Cookie[] cookies) { + return Stream.of(cookies) + .filter(cookie -> cookie.getName().startsWith(_PK_ID_NAME)) + .map(cookie -> _pk_id.matcher(cookie.getValue())) + .filter(Matcher::find) + .findFirst() + .map(Matcher::group); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCookieSessionEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCookieSessionEnricher.java new file mode 100644 index 000000000000..dee3b34f4c85 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCookieSessionEnricher.java @@ -0,0 +1,36 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.util.Locale; + +import jakarta.servlet.http.Cookie; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * This class adds the {@code MATOMO_SESSID} cookie to the {@code MatomoRequestDetails} + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestCookieSessionEnricher implements MatomoRequestDetailsEnricher { + + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + Cookie[] cookies = usageEvent.getRequest().getCookies(); + if (cookies == null) { + return matomoRequestDetails; + } + for (Cookie cookie : cookies) { + if (cookie.getName().toLowerCase(Locale.ROOT).equalsIgnoreCase("MATOMO_SESSID")) { + return matomoRequestDetails.addCookie("MATOMO_SESSID", cookie.getValue()); + } + } + return matomoRequestDetails; + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCountryEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCountryEnricher.java new file mode 100644 index 000000000000..6b3d56be2314 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCountryEnricher.java @@ -0,0 +1,46 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.util.Locale; + +import jakarta.servlet.http.HttpServletRequest; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * This class adds the {@code country} parameter to the {@code MatomoRequestDetails} + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestCountryEnricher implements MatomoRequestDetailsEnricher { + + private static final Logger log = LogManager.getLogger(MatomoRequestCountryEnricher.class); + + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + return matomoRequestDetails.addParameter("country", getCountry(usageEvent.getRequest())); + } + + private String getCountry(HttpServletRequest request) { + String country = ""; + if (request != null) { + try { + Locale locale = request.getLocale(); + if (locale != null) { + country = locale.getCountry().toLowerCase(); + } + } catch (Exception e) { + log.error("Cannot get locale of request!", e); + } + } + return country; + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCustomCookiesEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCustomCookiesEnricher.java new file mode 100644 index 000000000000..7f658a2776cb --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCustomCookiesEnricher.java @@ -0,0 +1,58 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.util.Set; + +import jakarta.servlet.http.Cookie; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * This class extends the {@code MatomoRequestDetailsEnricher} interface and provides a concrete implementation + * to enrich the {@code MatomoRequestDetails} with custom cookies from the {@code UsageEvent}. + *
customCookies; + + public MatomoRequestCustomCookiesEnricher(String customCookies) { + this.customCookies = Set.of(customCookies.split(",")); + } + + /** + * Enriches the {@code MatomoRequestDetails} with custom cookies from the {@code UsageEvent}. + * + * @param usageEvent The {@code UsageEvent} containing the request. + * @param matomoRequestDetails The {@code MatomoRequestDetails} to be enriched. + * @return The enriched {@code MatomoRequestDetails}. + */ + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + Cookie[] cookies = usageEvent.getRequest().getCookies(); + if (cookies == null) { + return matomoRequestDetails; + } + for (Cookie cookie : cookies) { + String cookieName = cookie.getName(); + String baseName = null; + if (cookieName != null && cookieName.contains(".")) { + baseName = cookieName.substring(0, cookieName.indexOf(".")); + } + if (baseName != null && customCookies.contains(baseName)) { + matomoRequestDetails.addCookie(cookieName, cookie.getValue()); + } + } + return matomoRequestDetails; + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCustomVariablesEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCustomVariablesEnricher.java new file mode 100644 index 000000000000..501f353a58a2 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestCustomVariablesEnricher.java @@ -0,0 +1,119 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.StringTokenizer; +import java.util.stream.Collectors; + +import jakarta.servlet.http.Cookie; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * This class adds the {@code _cvar} parameter to the {@code MatomoRequestDetails} + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestCustomVariablesEnricher implements MatomoRequestDetailsEnricher { + + public static final String VARIABLE_PATTERN = "\"{0}\":[\"{1}\",\"{2}\"]"; + public static final String PK_CVAR = "_pk_cvar"; + + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + Cookie[] cookies = usageEvent.getRequest().getCookies(); + if (cookies == null) { + return matomoRequestDetails; + } + for (Cookie cookie : cookies) { + String cookieName = cookie.getName(); + String baseName = null; + if (cookieName != null && cookieName.contains(".")) { + baseName = cookieName.substring(0, cookieName.indexOf(".")); + } + if (baseName != null && baseName.toLowerCase(Locale.ROOT).startsWith(PK_CVAR)) { + return addCustomVariables(matomoRequestDetails, cookie.getValue()); + } + } + return matomoRequestDetails; + } + + private static MatomoRequestDetails addCustomVariables( + MatomoRequestDetails matomoRequestDetails, String customVariables + ) { + Map> parsedCustomVariables = parse(customVariables); + if (parsedCustomVariables == null) { + return matomoRequestDetails; + } + + String variables = + parsedCustomVariables.entrySet() + .stream() + .map(entry -> + MessageFormat.format( + VARIABLE_PATTERN, + entry.getKey(), + entry.getValue().getKey(), entry.getValue().getValue() + ) + ) + .collect(Collectors.joining(",")); + + if (variables.isEmpty()) { + return matomoRequestDetails; + } + return matomoRequestDetails.addParameter("_cvar", "{" + variables + "}"); + } + + /** + * Parses a JSON representation of custom variables. + * + *

The format is as follows: {@code {"1":["key1","value1"],"2":["key2","value2"]}} + * + *

Example: {@code {"1":["OS","Windows"],"2":["Browser","Firefox"]}} + * + *

This is mainly used to parse the custom variables from the cookie. + * + * @param value The JSON representation of the custom variables to parse or null + * @return The parsed custom variables or null if the given value is null or empty + */ + public static Map> parse(String value) { + if (value == null || value.isEmpty()) { + return null; + } + + StringTokenizer tokenizer = new StringTokenizer(value, ":{}\""); + + Integer key = null; + String customVariableKey = null; + String customVariableValue = null; + Map> parsedVariables = new HashMap<>(); + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken().trim(); + if (!token.isEmpty()) { + if (token.matches("\\d+")) { + key = Integer.parseInt(token); + } else if (token.startsWith("[") && key != null) { + customVariableKey = tokenizer.nextToken(); + tokenizer.nextToken(); + customVariableValue = tokenizer.nextToken(); + } else if (key != null && customVariableKey != null && customVariableValue != null) { + parsedVariables.put(key, Map.entry(customVariableKey, customVariableValue)); + } else if (token.equals(",")) { + key = null; + customVariableKey = null; + customVariableValue = null; + } + } + } + return parsedVariables; + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestDetailsEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestDetailsEnricher.java new file mode 100644 index 000000000000..f868ad8b550b --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestDetailsEnricher.java @@ -0,0 +1,50 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * This class encapsulate a functional interface that will be used to enrich the {@code MatomoRequestDetails} + * with parameters. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +@FunctionalInterface +public interface MatomoRequestDetailsEnricher { + + /** + * Enriches the provided MatomoRequestDetails with additional parameters based on the UsageEvent. + * + * @param usageEvent The usage event containing information to enrich the request details + * @param matomoRequestDetails The request details object to be enriched with additional parameters + * @return The enriched MatomoRequestDetails object + */ + MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails); + + /** + * Composes this enricher with another enricher to create a new enricher that applies both in sequence. + * The provided enricher is applied first, followed by this enricher. + * + * @param enricher The enricher to compose with this one + * @return A new enricher that applies both enrichers in sequence + */ + default MatomoRequestDetailsEnricher compose(MatomoRequestDetailsEnricher enricher) { + return (usageEvent, details) -> enrich(usageEvent, enricher.enrich(usageEvent, details)); + } + + /** + * Creates an empty enricher that returns the input MatomoRequestDetails unchanged. + * + * @return An enricher that performs no modifications to the input + */ + static MatomoRequestDetailsEnricher empty() { + return ((usageEvent, matomoRequestDetails) -> matomoRequestDetails); + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestDetailsEnricherFactory.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestDetailsEnricherFactory.java new file mode 100644 index 000000000000..d8d11947cef0 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestDetailsEnricherFactory.java @@ -0,0 +1,147 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.sql.SQLException; +import java.util.Optional; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.content.DSpaceObject; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.core.Constants; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.usage.UsageEvent; + +/** + * This factory contains all the standard enricher that will add those parameters to the + * {@code MatomoRequestDetails} request + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestDetailsEnricherFactory { + + private MatomoRequestDetailsEnricherFactory() {} + + private static final Logger log = LogManager.getLogger(MatomoRequestDetailsEnricherFactory.class); + + /** + * Adds the {@code user-agent} to the Matomo request + * @return + */ + public static MatomoRequestDetailsEnricher userAgentEnricher() { + return (usageEvent, details) -> + details.addParameter( + "ua", + StringUtils.defaultIfBlank(usageEvent.getRequest().getHeader("USER-AGENT"), "") + ); + } + + /** + * Adds the {@code action_name} to the request + * @return + */ + public static MatomoRequestDetailsEnricher actionNameEnricher() { + return (usageEvent, details) -> + details.addParameter( + "action_name", + StringUtils.defaultIfBlank(actionName(usageEvent), "") + ); + } + + /** + * Adds the {@code url} of the tracked element. + * @return + */ + public static MatomoRequestDetailsEnricher urlEnricher() { + return (usageEvent, details) -> + details.addParameter("url", url(usageEvent.getObject())); + } + + /** + * Adds the {@code download} link of the tracked bitstream + * @return + */ + public static MatomoRequestDetailsEnricher downloadEnricher() { + return (usageEvent, details) -> + details.addParameter( + "download", + Optional.ofNullable(usageEvent.getObject()) + .filter(dso -> dso.getType() == Constants.BITSTREAM) + .map(MatomoRequestDetailsEnricherFactory::url) + .filter(StringUtils::isNotEmpty) + .map(url -> url + "/download") + .orElse("") + ); + } + + + /** + * Factory method that creates a new instance of MatomoRequestDetailsEnricher specialized for + * tracker identification enrichment. + * + * This enricher is responsible for adding tracker identification parameters to Matomo tracking + * requests, ensuring proper tracking attribution in the Matomo analytics system. + * + * @return A MatomoRequestDetailsEnricher instance configured to add tracker identification + * parameters to Matomo requests. The returned instance is specifically a + * MatomoRequestTrackerIdentifierParamEnricher. + * @see MatomoRequestTrackerIdentifierParamEnricher + * @see MatomoRequestDetailsEnricher + */ + public static MatomoRequestDetailsEnricher trackerIdentifierEnricher() { + return new MatomoRequestTrackerIdentifierParamEnricher(); + } + + private static String url(DSpaceObject dso) { + if (dso == null) { + return ""; + } + String baseUrl = dspaceUrl(); + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + } + return switch (dso.getType()) { + case Constants.BITSTREAM -> baseUrl + "/bitstreams/" + dso.getID(); + case Constants.ITEM -> baseUrl + "/items/" + dso.getID(); + default -> ""; + }; + } + + private static String dspaceUrl() { + return DSpaceServicesFactory.getInstance() + .getConfigurationService().getProperty("dspace.ui.url"); + } + + private static String actionName(UsageEvent ue) { + if (ue == null || ue.getObject() == null) { + return null; + } + try { + if (ue.getObject().getType() == Constants.BITSTREAM) { + // For a bitstream download we really want to know the title of the owning item + // rather than the bitstream name. + return ContentServiceFactory.getInstance() + .getDSpaceObjectService(ue.getObject()) + .getParentObject(ue.getContext(), ue.getObject()) + .getName(); + } else { + return ue.getObject().getName(); + } + } catch (SQLException e) { + // This shouldn't merit interrupting the user's transaction so log the error and continue. + log.error("Error in Matomo Analytics recording - can't determine ParentObjectName for bitstream {}", + ue.getObject().getID(), e + ); + } + + return null; + + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestIpAddressEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestIpAddressEnricher.java new file mode 100644 index 000000000000..7be91a242deb --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestIpAddressEnricher.java @@ -0,0 +1,37 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import org.apache.commons.lang3.StringUtils; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.service.ClientInfoService; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Adds the IP address of the client to the {@code MatomoRequestDetails}. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestIpAddressEnricher implements MatomoRequestDetailsEnricher { + + private final ClientInfoService clientInfoService; + + public MatomoRequestIpAddressEnricher(@Autowired ClientInfoService clientInfoService) { + this.clientInfoService = clientInfoService; + } + + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + String clientIp = ""; + if (usageEvent != null && usageEvent.getRequest() != null) { + clientIp = clientInfoService.getClientIp(usageEvent.getRequest()); + } + return matomoRequestDetails.addParameter("cip", StringUtils.defaultIfEmpty(clientIp, "")); + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestTrackerIdentifierParamEnricher.java b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestTrackerIdentifierParamEnricher.java new file mode 100644 index 000000000000..72ca96c49d0f --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/factory/MatomoRequestTrackerIdentifierParamEnricher.java @@ -0,0 +1,57 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.factory; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; + +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.usage.UsageEvent; + +/** + * Enricher implementation that extracts and validates a tracker identifier from request parameters + * and adds it to the Matomo request details. + * + *

This enricher looks for a 'trackerId' parameter in the usage event request parameters. + * If found and the value matches the expected 16-character hexadecimal format, it will be added + * to the Matomo request details as a visitor identifier.

+ * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + */ +public class MatomoRequestTrackerIdentifierParamEnricher implements MatomoRequestDetailsEnricher { + + public static final String ID_REGEX = "(^([a-f]|[0-9]){16})"; + static final String TRACKER_ID = "trackerId"; + static final String VISITOR_IDENTIFIER = "_id"; + + @Override + public MatomoRequestDetails enrich(UsageEvent usageEvent, MatomoRequestDetails matomoRequestDetails) { + if (usageEvent == null || usageEvent.getRequest() == null) { + return matomoRequestDetails; + } + + Map parameterMap = usageEvent.getRequest().getParameterMap(); + Optional validParameterValue = getValidParameterValue(parameterMap); + if (validParameterValue.isEmpty()) { + return matomoRequestDetails; + } + + return matomoRequestDetails.addParameter(VISITOR_IDENTIFIER, validParameterValue.get()); + } + + private Optional getValidParameterValue(Map parameterMap) { + return parameterMap.entrySet().stream() + .filter(entry -> entry.getKey().equals(TRACKER_ID)) + .flatMap(entry -> + Arrays.stream(entry.getValue()) + .filter(value -> value.matches(ID_REGEX)) + ) + .findFirst(); + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/model/MatomoCookieConverter.java b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoCookieConverter.java new file mode 100644 index 000000000000..14dffa4d3566 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoCookieConverter.java @@ -0,0 +1,75 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import java.text.MessageFormat; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class is used to convert the {@code MatomoRequestDetails} cookies + * to the Matomo format. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoCookieConverter { + + private MatomoCookieConverter() { + } + + /** + * Converts a single MatomoRequestDetails object's cookies into a Matomo-formatted string + * + * @param matomoRequestDetails The MatomoRequestDetails object containing cookies to convert + * @return A string containing the cookies in key=value format, separated by semicolons + */ + public static String convert(MatomoRequestDetails matomoRequestDetails) { + return matomoRequestDetails.cookies + .entrySet() + .stream() + .map((entry) -> + MessageFormat.format("{0}={1}", entry.getKey(), entry.getValue()) + ) + .collect(Collectors.joining(";")); + } + + /** + * Converts a list of MatomoRequestDetails objects' cookies into a single Matomo-formatted string. + * If multiple cookies have the same key, the last value will be used. + * + * @param matomoRequestDetails List of MatomoRequestDetails objects containing cookies to convert + * @return A string containing the merged cookies in key=value format, separated by semicolons. + * Returns empty string if input is null or empty. + */ + public static String convert(List matomoRequestDetails) { + if (matomoRequestDetails == null || matomoRequestDetails.isEmpty()) { + return ""; + } + Map reducedCookies = + matomoRequestDetails.stream() + .flatMap(details -> details.cookies.entrySet().stream()) + .collect( + Collectors.groupingBy( + Map.Entry::getKey, + Collectors.mapping( + Map.Entry::getValue, + Collectors.reducing( + "", + (a, b) -> b + ) + ) + ) + ); + return reducedCookies.entrySet().stream() + .map((entry) -> + MessageFormat.format("{0}={1}", entry.getKey(), entry.getValue()) + ) + .collect(Collectors.joining(";")); + } +} \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetails.java b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetails.java new file mode 100644 index 000000000000..596c62e45ff4 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetails.java @@ -0,0 +1,46 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import java.util.HashMap; +import java.util.Map; + +/** + * Encapsulates the details of a single request + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestDetails { + + final Map parameters = new HashMap<>(); + final Map cookies = new HashMap<>(); + + /** + * Adds a parameter key-value pair to the request details + * + * @param key The parameter key + * @param value The parameter value + * @return The current MatomoRequestDetails instance for method chaining + */ + public MatomoRequestDetails addParameter(String key, String value) { + parameters.put(key, value); + return this; + } + + /** + * Adds a cookie key-value pair to the request details + * @param key The cookie key + * @param value The cookie value + * @return The current MatomoRequestDetails instance for method chaining + */ + public MatomoRequestDetails addCookie(String key, String value) { + cookies.put(key, value); + return this; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsBuilder.java b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsBuilder.java new file mode 100644 index 000000000000..38149b72da3e --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsBuilder.java @@ -0,0 +1,61 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import java.util.List; + +import org.dspace.matomo.factory.MatomoRequestDetailsEnricher; +import org.dspace.usage.UsageEvent; + +/** + * This builder can be used to create a proper request using the configured {@code List} + * and the proper {@code siteId}. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestDetailsBuilder { + + final String siteId; + final List enrichers; + + /** + * Constructs a new MatomoRequestDetailsBuilder with the specified enrichers and site ID. + * + * @param enrichers List of MatomoRequestDetailsEnricher objects to enrich the request details + * @param siteId The Matomo site ID to be used for tracking + */ + MatomoRequestDetailsBuilder( + List enrichers, + String siteId + ) { + this.enrichers = enrichers; + this.siteId = siteId; + } + + /** + * Builds a MatomoRequestDetails object for the given usage event. + * This method initializes basic tracking parameters and applies all configured enrichers. + * + * @param usageEvent The usage event to build request details for + * @return MatomoRequestDetails object containing all tracking parameters + */ + public MatomoRequestDetails build(UsageEvent usageEvent) { + MatomoRequestDetails requestDetails = new MatomoRequestDetails(); + + requestDetails.addParameter("idsite", siteId) + .addParameter("rec", "1") + .addParameter("cookie", "1") + .addParameter("apiv", "1"); + + return enrichers.stream() + .reduce(MatomoRequestDetailsEnricher.empty(), MatomoRequestDetailsEnricher::compose) + .enrich(usageEvent, requestDetails); + } + + +} \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsListConverter.java b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsListConverter.java new file mode 100644 index 000000000000..63abaa86c486 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsListConverter.java @@ -0,0 +1,68 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.Collection; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * This class will be used to convert each {@code MatomoRequestDetails} into a proper URL that will be serialized + * into a valid JSON. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestDetailsListConverter> extends JsonSerializer { + + private static final Logger log = LogManager.getLogger(MatomoRequestDetailsListConverter.class); + // each request will be mapped to: ?parameter1=value1¶meter2=value2 + private final String requestTemplate = "?{0}"; + // each key-value will be mapped to: key=value + private final String keyValueTemplate = "{0}={1}"; + + @Override + public void serialize(T requests, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + gen.writeObject( + requests.stream() + .map(this::mapRequest) + .collect(Collectors.toList()) + ); + } + + private String mapRequest(MatomoRequestDetails request) { + return MessageFormat.format(requestTemplate, getRequestURL(request)); + } + + /** + * Converts the request parameters into a URL query string format. + * The resulting format will be: param1=value1¶m2=value2¶m3=value3 + * where each parameter-value pair is joined with '=' and multiple pairs are joined with '&'. + * Empty parameters or values are filtered out. + * + * @param request The MatomoRequestDetails containing the parameters to convert + * @return A URL query string containing the formatted parameters + */ + private String getRequestURL(MatomoRequestDetails request) { + return request.parameters.entrySet() + .stream() + .filter(entry -> StringUtils.isNoneEmpty(entry.getValue(), entry.getKey())) + .map(entry -> + MessageFormat.format(keyValueTemplate, entry.getKey(), entry.getValue()) + ) + .collect(Collectors.joining("&")); + } +} diff --git a/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsSplitter.java b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsSplitter.java new file mode 100644 index 000000000000..a6148bb6cb9d --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/matomo/model/MatomoRequestDetailsSplitter.java @@ -0,0 +1,42 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class is used to split a list of {@code MatomoRequestDetails} into a map + * of {@code List}.
+ * The key of the map is the value of the parameter named "_id", or "default" if the "_id" is not set. + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + **/ +public class MatomoRequestDetailsSplitter { + + /** + * Private constructor to prevent instantiation of utility class. + */ + private MatomoRequestDetailsSplitter () { } + + /** + * Splits a list of MatomoRequestDetails into a map grouped by their "_id" parameter. + * + * @param details The list of MatomoRequestDetails to split + * @return A Map where the key is the "_id" parameter value (or "default" if not set) and + * the value is a List of MatomoRequestDetails with that "_id" + */ + public static Map> split(List details) { + return details.stream().collect( + Collectors.groupingBy( + detail -> detail.parameters.getOrDefault("_id","default")) + ); + } + +} \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java index c89e5d7a54d1..59131e78a36a 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/BitstreamStorageServiceImpl.java @@ -402,6 +402,25 @@ public void migrate(Context context, Integer assetstoreSource, Integer assetstor while (allBitstreamsInSource.hasNext()) { Bitstream bitstream = allBitstreamsInSource.next(); + + // UMD Customization + // This customization was added to support migrating the asset store + // files in the Kubernetes "sandbox", "test" and "qa" namespaces to + // AWS S3 and can be removed after the AWS S3 migration is complete. + BitStoreService bitstore = getStore(bitstream.getStoreNumber()); + if (bitstore instanceof DSBitStoreService) { + DSBitStoreService dsBitstoreService = (DSBitStoreService) bitstore; + if (!dsBitstoreService.exists(bitstream)) { + log.info("Skipping bitstream:" + bitstream.getID() + + " from assetstore[" + assetstoreSource + "] " + + "Name:" + bitstream.getName() + + ", SizeBytes:" + bitstream.getSizeBytes() + + " because it does not exist in the asset store!"); + continue; + } + } + // End UMD Customization + log.info("Copying bitstream:" + bitstream .getID() + " from assetstore[" + assetstoreSource + "] to assetstore[" + assetstoreDestination + "] " + "Name:" + bitstream @@ -423,7 +442,13 @@ public void migrate(Context context, Integer assetstoreSource, Integer assetstor //modulo if ((processedCounter % batchCommitSize) == 0) { log.info("Migration Commit Checkpoint: " + processedCounter); - context.dispatchEvents(); + // UMD Customization + // This change was provided to DSpace in Pull Request 10940 + // This customization markers can be removed once the + // application has been upgraded to a DSpace version containing + // the pull request. + context.commit(); + // End UMD Customizaton } } diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/DSBitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/DSBitStoreService.java index 6fef7365e482..feb2d6a5c2a8 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/DSBitStoreService.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/DSBitStoreService.java @@ -263,4 +263,18 @@ public File getBaseDir() { public void setBaseDir(File baseDir) { this.baseDir = baseDir; } + + // UMD Customization + // This customization was added to support migrating the asset store + // files in the Kubernetes "sandbox", "test" and "qa" namespaces to + // AWS S3 and can be removed after the AWS S3 migration is complete. + public boolean exists(Bitstream bitstream) + throws IOException { + File file = getFile(bitstream); + if (file == null) { + return false; + } + return file.exists(); + } + // End UMD Customization } diff --git a/dspace-api/src/test/data/dspaceFolder/config/hibernate.cfg.xml b/dspace-api/src/test/data/dspaceFolder/config/hibernate.cfg.xml new file mode 100644 index 000000000000..9d1ee9dac067 --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/hibernate.cfg.xml @@ -0,0 +1,108 @@ + + + + + + + validate + org.hibernate.tool.hbm2ddl.SingleLineSqlCommandExtractor + false + 20 + org.hibernate.context.internal.ThreadLocalSessionContext + + + false + + + true + true + true + + org.hibernate.cache.jcache.JCacheRegionFactory + + + org.ehcache.jsr107.EhcacheCachingProvider + + + ENABLE_SELECTIVE + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml new file mode 100644 index 000000000000..15bb3ef1580b --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/bitstore.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/core-dao-services.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/core-dao-services.xml new file mode 100644 index 000000000000..54cfb2df3427 --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/core-dao-services.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/core-services.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/core-services.xml new file mode 100644 index 000000000000..8d9c9d3dd336 --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/core-services.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + news-top.html + news-side.html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dspace-api/src/test/java/org/dspace/checker/ChecksumCheckerIT.java b/dspace-api/src/test/java/org/dspace/checker/ChecksumCheckerIT.java new file mode 100644 index 000000000000..173ffc79c476 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/checker/ChecksumCheckerIT.java @@ -0,0 +1,202 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.checker; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.checker.factory.CheckerServiceFactory; +import org.dspace.checker.service.ChecksumHistoryService; +import org.dspace.checker.service.MostRecentChecksumService; +import org.dspace.content.Bitstream; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.core.Context; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * UMD Customization + * + * A modified version of this class was provided to DSpace in + * Pull Request 10508. + * + * This class should be replaced with the DSpace version, once this application + * has been upgraded to a DSpace version containing the pull request. + */ +public class ChecksumCheckerIT extends AbstractIntegrationTestWithDatabase { + protected List bitstreams; + protected MostRecentChecksumService checksumService = + CheckerServiceFactory.getInstance().getMostRecentChecksumService(); + + @Before + public void setup() throws Exception { + context.turnOffAuthorisationSystem(); + + Community parentCommunity = CommunityBuilder.createCommunity(context).build(); + Collection collection = CollectionBuilder.createCollection(context, parentCommunity) + .build(); + Item item = ItemBuilder.createItem(context, collection).withTitle("Test item") + .build(); + + int numBitstreams = 3; + bitstreams = new ArrayList<>(); + for (int i = 0; i < numBitstreams; i++) { + String content = "Test bitstream " + i; + bitstreams.add( + BitstreamBuilder.createBitstream( + context, item, IOUtils.toInputStream(content, UTF_8) + ).build() + ); + } + + context.restoreAuthSystemState(); + + // Call the "updateMissingBitstreams" method so that the test bitstreams + // already have checksums in the past when CheckerCommand runs. + // Otherwise, the CheckerCommand will simply update the test + // bitstreams without going through the BitstreamDispatcher. + checksumService = CheckerServiceFactory.getInstance().getMostRecentChecksumService(); + checksumService.updateMissingBitstreams(context); + + // The "updateMissingBitstreams" method updates the test bitstreams in + // a random order. To verify that the expected bitstreams were + // processed, reset the timestamps so that the bitstreams are + // checked in a specific order (oldest first). + Date checksumInstant = Date.from(Instant.ofEpochMilli(0)); + for (Bitstream bitstream: bitstreams) { + MostRecentChecksum mrc = checksumService.findByBitstream(context, bitstream); + mrc.setProcessStartDate(checksumInstant); + mrc.setProcessEndDate(checksumInstant); + checksumInstant = new Date(checksumInstant.getTime() + 10000); + } + context.commit(); + } + + @After + public void cleanUp() throws SQLException { + // Need to clean up ChecksumHistory because of a referential integrity + // constraint violation between the most_recent_checksum table and + // bitstream tables + ChecksumHistoryService checksumHistoryService = CheckerServiceFactory.getInstance().getChecksumHistoryService(); + + for (Bitstream bitstream: bitstreams) { + checksumHistoryService.deleteByBitstream(context, bitstream); + } + } + + @Test + public void testChecksumsRecordedWhenProcesingIsInterrupted() throws SQLException { + CheckerCommand checker = new CheckerCommand(context); + + // The start date to use for the checker process + Date checkerStartDate = Date.from(Instant.now()); + + // Verify that all checksums are before the checker start date + for (Bitstream bitstream: bitstreams) { + MostRecentChecksum checksum = checksumService.findByBitstream(context, bitstream); + Date lastChecksumDate = checksum.getProcessStartDate(); + assertTrue("lastChecksumDate (" + lastChecksumDate + ") <= checkerStartDate (" + checkerStartDate + ")", + lastChecksumDate.before(checkerStartDate)); + } + + // Dispatcher that throws an exception when a third bitstream is + // retrieved. + BitstreamDispatcher dispatcher = new ExpectionThrowingDispatcher( + context, checkerStartDate, false, 2); + checker.setDispatcher(dispatcher); + + + // Run the checksum checker + checker.setProcessStartDate(checkerStartDate); + try { + checker.process(); + fail("SQLException should have been thrown"); + } catch (SQLException sqle) { + // Rollback any pending transaction + context.rollback(); + } + + // Verify that the checksums of the first two bitstreams (that were + // processed before the exception) have been successfully recorded in + // the database, while the third bitstream was not updated. + int bitstreamCount = 0; + for (Bitstream bitstream: bitstreams) { + MostRecentChecksum checksum = checksumService.findByBitstream(context, bitstream); + Date lastChecksumDate = checksum.getProcessStartDate(); + + bitstreamCount = bitstreamCount + 1; + if (bitstreamCount <= 2) { + assertTrue("lastChecksumDate (" + lastChecksumDate + ") <= checkerStartDate (" + checkerStartDate + ")", + lastChecksumDate.after(checkerStartDate)); + } else { + assertTrue("lastChecksumDate (" + lastChecksumDate + ") >= checkerStartDate (" + checkerStartDate + ")", + lastChecksumDate.before(checkerStartDate)); + } + } + } + + /** + * Subclass of SimpleDispatcher that only allows a limited number of "next" + * class before throwing a SQLException. + */ + class ExpectionThrowingDispatcher extends SimpleDispatcher { + // The number of "next" calls to allow before throwing a SQLException + protected int maxNextCalls; + + // The number of "next" method calls seen so far. + protected int numNextCalls = 0; + + /** + * Constructor. + * + * @param context Context + * @param startTime timestamp for beginning of checker process + * @param looping indicates whether checker should loop infinitely + * through most_recent_checksum table + * @param maxNextCalls the number of "next" method calls to allow before + * throwing a SQLException. + */ + public ExpectionThrowingDispatcher(Context context, Date startTime, boolean looping, int maxNextCalls) { + super(context, startTime, looping); + this.maxNextCalls = maxNextCalls; + } + + /** + * Selects the next candidate bitstream. + * + * After "maxNextClass" number of calls, this method throws a + * SQLException. + * + * @throws SQLException if database error + */ + @Override + public synchronized Bitstream next() throws SQLException { + numNextCalls = numNextCalls + 1; + if (numNextCalls > maxNextCalls) { + throw new SQLException("Max 'next' method calls exceeded"); + } + return super.next(); + } + } +} diff --git a/dspace-api/src/test/java/org/dspace/matomo/MatomoAsyncBulkRequestHandlerTest.java b/dspace-api/src/test/java/org/dspace/matomo/MatomoAsyncBulkRequestHandlerTest.java new file mode 100644 index 000000000000..ae54ff92fea4 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/MatomoAsyncBulkRequestHandlerTest.java @@ -0,0 +1,111 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import java.util.List; + +import org.dspace.AbstractUnitTest; +import org.dspace.matomo.client.MatomoClient; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.matomo.model.MatomoRequestDetailsBuilder; +import org.dspace.usage.UsageEvent; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +public class MatomoAsyncBulkRequestHandlerTest extends AbstractUnitTest { + + @Mock + MatomoRequestDetailsBuilder builder; + @Mock + MatomoClient matomoClient; + + MatomoAsyncBulkRequestHandler matomoAsyncDequeHandler; + + @Before + public void setUp() throws Exception { + matomoAsyncDequeHandler = + new MatomoAsyncBulkRequestHandler(builder, matomoClient, 5); + } + + @Test + public void testNullEvent() { + matomoAsyncDequeHandler.handleEvent(null); + Mockito.verifyNoInteractions(matomoClient); + Mockito.verifyNoInteractions(builder); + } + + @Test + public void testSingleRequestHigherCapacity() { + UsageEvent usageEvent = Mockito.mock(UsageEvent.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + Mockito.when(builder.build(usageEvent)).thenReturn(matomoRequestDetails); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + + Mockito.verifyNoInteractions(matomoClient); + } + + + @Test + public void testSingleRequestCapacity() { + matomoAsyncDequeHandler = + new MatomoAsyncBulkRequestHandler(builder, matomoClient, 1); + + UsageEvent usageEvent = Mockito.mock(UsageEvent.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + Mockito.when(builder.build(usageEvent)).thenReturn(matomoRequestDetails); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + + Mockito.verify(matomoClient, Mockito.times(1)) + .sendDetails(Mockito.any(List.class)); + } + + + @Test + public void testTwoRequestsTwoCapacity() { + matomoAsyncDequeHandler = + new MatomoAsyncBulkRequestHandler(builder, matomoClient, 3); + + UsageEvent usageEvent = Mockito.mock(UsageEvent.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + Mockito.when(builder.build(usageEvent)).thenReturn(matomoRequestDetails); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + Mockito.verify(matomoClient, Mockito.never()).sendDetails(Mockito.any(List.class)); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(Mockito.any(List.class)); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(Mockito.any(List.class)); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + Mockito.verify(matomoClient, Mockito.times(2)).sendDetails(Mockito.any(List.class)); + } + + @Test + public void testManualSendEvents() { + UsageEvent usageEvent = Mockito.mock(UsageEvent.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + Mockito.when(builder.build(usageEvent)).thenReturn(matomoRequestDetails); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + Mockito.verify(matomoClient, Mockito.never()).sendDetails(Mockito.any(List.class)); + + matomoAsyncDequeHandler.handleEvent(usageEvent); + Mockito.verify(matomoClient, Mockito.never()).sendDetails(Mockito.any(List.class)); + + matomoAsyncDequeHandler.sendEvents(); + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(Mockito.any(List.class)); + } + + +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/MatomoEventListenerTest.java b/dspace-api/src/test/java/org/dspace/matomo/MatomoEventListenerTest.java new file mode 100644 index 000000000000..e60f2cfba1f6 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/MatomoEventListenerTest.java @@ -0,0 +1,120 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import java.sql.SQLException; +import java.util.List; +import java.util.Set; + +import org.dspace.AbstractUnitTest; +import org.dspace.content.Bitstream; +import org.dspace.content.Item; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Constants; +import org.dspace.services.ConfigurationService; +import org.dspace.usage.UsageEvent; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +public class MatomoEventListenerTest extends AbstractUnitTest { + + @Mock + MatomoAsyncBulkRequestHandler matomoHandler1; + @Mock + MatomoSyncEventHandler matomoHandler2; + @Mock + ConfigurationService configurationService; + @Mock + BitstreamService bitstreamService; + + MatomoEventListener matomoEventListener; + + @Before + public void setUp() throws Exception { + matomoEventListener = + new MatomoEventListener(List.of(matomoHandler1, matomoHandler2), configurationService, bitstreamService); + } + + @Test + public void testDisabledMatomo() { + UsageEvent event = Mockito.mock(UsageEvent.class); + + matomoEventListener.receiveEvent(event); + + Mockito.verifyNoInteractions(matomoHandler1); + Mockito.verifyNoInteractions(matomoHandler2); + } + + + @Test + public void testDontHandleGenericViewEventWithMatomoEnabled() { + UsageEvent event = Mockito.mock(UsageEvent.class); + Mockito.when(event.getAction()).thenReturn(UsageEvent.Action.VIEW); + Mockito.when(event.getObject()).thenReturn(Mockito.spy(Item.class)); + + Mockito.when(configurationService.getBooleanProperty("matomo.enabled", false)) + .thenReturn(true); + + matomoEventListener.receiveEvent(event); + + Mockito.verifyNoInteractions(matomoHandler1); + Mockito.verifyNoInteractions(matomoHandler2); + } + + + @Test + public void testHandleBitstreamViewEvent() throws SQLException { + // mock event + UsageEvent event = Mockito.mock(UsageEvent.class); + Mockito.when(event.getAction()).thenReturn(UsageEvent.Action.VIEW); + + // mock bitstream + Bitstream bitstream = Mockito.spy(Bitstream.class); + Mockito.when( + bitstreamService.isInBundle( + Mockito.eq(bitstream), + Mockito.eq(Set.of(Constants.CONTENT_BUNDLE_NAME)) + ) + ).thenReturn(true); + + Mockito.when(event.getObject()).thenReturn(bitstream); + + // mock configuration + Mockito.when(configurationService.getBooleanProperty(Mockito.eq("matomo.enabled"), Mockito.eq(false))) + .thenReturn(true); + Mockito.when(configurationService.getArrayProperty(Mockito.eq("matomo.track.bundles"), Mockito.any())) + .thenReturn(new String[] { }); + + matomoEventListener.receiveEvent(event); + + Mockito.verifyNoInteractions(matomoHandler1); + Mockito.verifyNoInteractions(matomoHandler2); + + // none bundle, will skip processing + Mockito.when(configurationService.getArrayProperty(Mockito.eq("matomo.track.bundles"), Mockito.any())) + .thenReturn(new String[] {"none"}); + + matomoEventListener.receiveEvent(event); + + Mockito.verifyNoMoreInteractions(matomoHandler1); + Mockito.verifyNoMoreInteractions(matomoHandler2); + + // default ( original bundle only ) then proceed with the invocation + Mockito.when(configurationService.getArrayProperty(Mockito.eq("matomo.track.bundles"), Mockito.any())) + .thenReturn(new String[] { Constants.CONTENT_BUNDLE_NAME }); + + matomoEventListener.receiveEvent(event); + + Mockito.verify(matomoHandler1, Mockito.times(1)).handleEvent(event); + Mockito.verify(matomoHandler2, Mockito.times(1)).handleEvent(event); + + } + +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/MatomoSyncEventHandlerTest.java b/dspace-api/src/test/java/org/dspace/matomo/MatomoSyncEventHandlerTest.java new file mode 100644 index 000000000000..ad3c1eff19a0 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/MatomoSyncEventHandlerTest.java @@ -0,0 +1,72 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo; + +import org.dspace.AbstractUnitTest; +import org.dspace.matomo.client.MatomoClient; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.dspace.matomo.model.MatomoRequestDetailsBuilder; +import org.dspace.usage.UsageEvent; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; + +public class MatomoSyncEventHandlerTest extends AbstractUnitTest { + + @Mock + MatomoClient matomoClient; + @Mock + MatomoRequestDetailsBuilder builder; + @InjectMocks + MatomoSyncEventHandler matomoSyncEventHandler; + + @Test + public void testNullEvent() { + matomoSyncEventHandler.handleEvent(null); + Mockito.verifyNoInteractions(builder); + Mockito.verifyNoInteractions(matomoClient); + } + + @Test + public void testSingleEvent() { + UsageEvent usageEvent = Mockito.mock(UsageEvent.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + Mockito.when(builder.build(usageEvent)).thenReturn(matomoRequestDetails); + + matomoSyncEventHandler.handleEvent(usageEvent); + + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(matomoRequestDetails); + Mockito.verifyNoMoreInteractions(matomoClient); + } + + + @Test + public void testMultipleEvents() { + UsageEvent usageEvent1 = Mockito.mock(UsageEvent.class); + UsageEvent usageEvent2 = Mockito.mock(UsageEvent.class); + UsageEvent usageEvent3 = Mockito.mock(UsageEvent.class); + MatomoRequestDetails matomoRequestDetails1 = new MatomoRequestDetails(); + MatomoRequestDetails matomoRequestDetails2 = new MatomoRequestDetails(); + MatomoRequestDetails matomoRequestDetails3 = new MatomoRequestDetails(); + + Mockito.when(builder.build(usageEvent1)).thenReturn(matomoRequestDetails1); + Mockito.when(builder.build(usageEvent2)).thenReturn(matomoRequestDetails2); + Mockito.when(builder.build(usageEvent3)).thenReturn(matomoRequestDetails3); + + matomoSyncEventHandler.handleEvent(usageEvent1); + matomoSyncEventHandler.handleEvent(usageEvent2); + matomoSyncEventHandler.handleEvent(usageEvent3); + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(matomoRequestDetails1); + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(matomoRequestDetails2); + Mockito.verify(matomoClient, Mockito.times(1)).sendDetails(matomoRequestDetails3); + Mockito.verifyNoMoreInteractions(matomoClient); + } + + +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/client/MatomoAbstractClientTest.java b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoAbstractClientTest.java new file mode 100644 index 000000000000..bf4c22f59827 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoAbstractClientTest.java @@ -0,0 +1,151 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.net.HttpURLConnection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.dspace.AbstractUnitTest; +import org.junit.Test; + +public class MatomoAbstractClientTest extends AbstractUnitTest { + + private static class MockHttpURLConnection extends HttpURLConnection { + private Map requestProperties = new HashMap<>(); + + protected MockHttpURLConnection() { + super(null); + } + + @Override + public void setRequestProperty(String key, String value) { + requestProperties.put(key, value); + } + + @Override + public String getRequestProperty(String key) { + return requestProperties.get(key); + } + + // Stub implementations for abstract methods + @Override + public void disconnect() { + + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public void connect() { + + } + } + + /** + * Test case for addCookies method when a single cookie is provided. + * This test verifies that the method correctly sets the Cookie request property + * when given a map with a single cookie, and the resulting cookie string + * does not end with a semicolon. + */ + @Test + public void test_addCookies_singleCookie() { + HttpURLConnection connection = mock(HttpURLConnection.class); + Map cookies = new LinkedHashMap<>(); + cookies.put("testCookie", "testValue"); + + MatomoAbstractClient.addCookies(connection, cookies); + + verify(connection).setRequestProperty("Cookie", "testCookie=testValue"); + } + + /** + * Test case for static void addCookies(HttpURLConnection connection, Map cookies) + * This test verifies that when given a non-null but empty cookies map, + * no request property is set on the connection. + */ + @Test + public void test_addCookies_withEmptyCookiesMap() { + HttpURLConnection mockConnection = mock(HttpURLConnection.class); + Map cookies = new LinkedHashMap<>(); + + MatomoAbstractClient.addCookies(mockConnection, cookies); + + verify(mockConnection, never()).setRequestProperty(anyString(), anyString()); + } + + /** + * Tests the addCookies method when cookies are present and added successfully. + * Verifies that the Cookie request property is set correctly with multiple cookies. + */ + @Test + public void test_addCookies_withMultipleCookies() { + HttpURLConnection mockConnection = new MockHttpURLConnection(); + Map cookies = new HashMap<>(); + cookies.put("cookie1", "value1"); + cookies.put("cookie2", "value2"); + + MatomoAbstractClient.addCookies(mockConnection, cookies); + + String cookieHeader = mockConnection.getRequestProperty("Cookie"); + assertThat("Cookie header should contain both cookies", + cookieHeader, + allOf( + notNullValue(), + containsString("cookie1=value1"), + containsString("cookie2=value2") + ) + ); + } + + /** + * Tests the addCookies method with a null cookies map. + * This scenario is explicitly handled in the method implementation. + * Expected behavior: No exception should be thrown, and no request property should be set. + */ + @Test + public void test_addCookies_withNullCookiesMap() { + HttpURLConnection connection = mock(HttpURLConnection.class); + Map cookies = null; + + MatomoAbstractClient.addCookies(connection, cookies); + + verify(connection, never()).setRequestProperty(anyString(), anyString()); + } + + /** + * Test case for addCookies method when cookies map is null but a default cookie is set. + * This test verifies that even when the cookies map is null, if a default cookie is present, + * it will be added to the connection's request property. + */ + @Test + public void test_addCookies_withNullMapAndDefaultCookie() { + HttpURLConnection connection = mock(HttpURLConnection.class); + Map cookies = null; + + MatomoAbstractClient.addCookies(connection, cookies); + + verify(connection, never()).setRequestProperty(eq("Cookie"), anyString()); + verify(connection, never()).getRequestProperty(eq("Cookie")); + } + +} diff --git a/dspace-api/src/test/java/org/dspace/matomo/client/MatomoClientImplTest.java b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoClientImplTest.java new file mode 100644 index 000000000000..88cf33db5035 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoClientImplTest.java @@ -0,0 +1,144 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import java.io.IOException; +import java.util.List; + +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.dspace.AbstractUnitTest; +import org.dspace.matomo.exception.MatomoClientException; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +/** + * Test for MatomoClientImplTest + * + * @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com) + */ +public class MatomoClientImplTest extends AbstractUnitTest { + + @Mock + CloseableHttpClient httpClient; + + @Mock + CloseableHttpResponse response; + + @Mock + MatomoRequestBuilder builder; + + @Mock + MatomoResponseReader reader; + + MatomoClientImpl matomoClient; + + @Before + public void setUp() throws Exception { + matomoClient = new MatomoClientImpl("testURL", "custom-token", builder, reader, httpClient); + } + + @Test + public void testNullRequest() { + this.matomoClient.sendDetails((List) null); + Mockito.verifyNoInteractions(httpClient); + Mockito.verifyNoInteractions(builder); + + this.matomoClient.sendDetails(); + Mockito.verifyNoInteractions(httpClient); + Mockito.verifyNoInteractions(builder); + } + + @Test + public void testEmptyRequest() { + this.matomoClient.sendDetails(List.of()); + Mockito.verifyNoInteractions(httpClient); + Mockito.verifyNoInteractions(builder); + + this.matomoClient.sendDetails(); + Mockito.verifyNoInteractions(httpClient); + Mockito.verifyNoInteractions(builder); + } + + + @Test + public void testSingleRequest() throws IOException { + + MatomoRequestDetails details = + new MatomoRequestDetails() + .addParameter("test1", "value1") + .addParameter("test2", "value2") + .addParameter("test3", "value3"); + + String json = + "{\"auth_token\": \"custom-token\", \"requests\": [\"?test1=value1&test2=value2&test3=value3\"]}"; + String jsonResponse = + "{\"status\": \"success\", \"tracked\": 1, \"invalid\": 0, \"invalid_indices\": []}"; + Mockito.when(builder.buildJSON(Mockito.any())).thenReturn(json); + StatusLine mock = Mockito.mock(StatusLine.class); + Mockito.when(mock.getStatusCode()).thenReturn(200); + Mockito.when(response.getStatusLine()).thenReturn(mock); + Mockito.when(response.getEntity()).thenReturn(new StringEntity(jsonResponse)); + Mockito.when(reader.fromJSON(jsonResponse)) + .thenReturn(new MatomoResponse("success", 1, 0, null)); + Mockito.when(this.httpClient.execute(Mockito.any(HttpPost.class))).thenReturn(response); + + this.matomoClient.sendDetails(List.of(details)); + Mockito.verify(this.httpClient, Mockito.times(1)).execute(Mockito.any(HttpPost.class)); + + this.matomoClient.sendDetails(details); + Mockito.verify(this.httpClient, Mockito.times(2)).execute(Mockito.any(HttpPost.class)); + } + + @Test(expected = MatomoClientException.class) + public void testFailSingleRequest() throws IOException { + + MatomoRequestDetails details = + new MatomoRequestDetails() + .addParameter("test1", "value1") + .addParameter("test2", "value2") + .addParameter("test3", "value3"); + + String json = + "{'auth_token': 'custom-token', 'requests': ['?test1=value1&test2=value2&test3=value3']}"; + StatusLine statusLine = Mockito.mock(StatusLine.class); + Mockito.when(builder.buildJSON(Mockito.any())).thenReturn(json); + Mockito.when(statusLine.getStatusCode()).thenReturn(500); + Mockito.when(response.getStatusLine()).thenReturn(statusLine); + Mockito.when(this.httpClient.execute(Mockito.any(HttpPost.class))).thenReturn(response); + + this.matomoClient.sendDetails(List.of(details)); + } + + + @Test(expected = MatomoClientException.class) + public void testExceptionRequest() throws IOException { + + MatomoRequestDetails details = + new MatomoRequestDetails() + .addParameter("test1", "value1") + .addParameter("test2", "value2") + .addParameter("test3", "value3"); + + String json = + "{'auth_token': 'custom-token', 'requests': ['?test1=value1&test2=value2&test3=value3']}"; + Mockito.when(builder.buildJSON(Mockito.any())).thenReturn(json); + Mockito.doThrow(IOException.class) + .when(this.httpClient) + .execute(Mockito.any(HttpPost.class)); + this.matomoClient.sendDetails(List.of(details)); + } + + +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/client/MatomoRequestBuilderTest.java b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoRequestBuilderTest.java new file mode 100644 index 000000000000..805248dfed40 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoRequestBuilderTest.java @@ -0,0 +1,154 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.emptyOrNullString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; + +import java.util.List; + +import org.dspace.AbstractUnitTest; +import org.dspace.matomo.model.MatomoRequestDetails; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +public class MatomoRequestBuilderTest extends AbstractUnitTest { + + MatomoRequestBuilder matomoRequestBuilder; + + @Before + public void setUp() throws Exception { + matomoRequestBuilder = new MatomoRequestBuilder(); + } + + @Test + public void testNull() throws JSONException { + matomoRequestBuilder = new MatomoRequestBuilder(); + String json = matomoRequestBuilder.buildJSON(null); + assertThat(json, emptyOrNullString()); + } + + @Test + public void testEmptyRequests() throws JSONException { + matomoRequestBuilder = new MatomoRequestBuilder(); + String json = matomoRequestBuilder.buildJSON(new MatomoBulkRequest("my-token", List.of())); + JSONObject jsonObject = new JSONObject(json); + assertThat(jsonObject.getString("token_auth"), is("my-token")); + + assertThat(jsonObject.has("requests"), is(true)); + + JSONArray requests = jsonObject.getJSONArray("requests"); + assertThat(requests, notNullValue()); + assertThat(requests.length(), is(0)); + } + + @Test + public void testNullRequests() throws JSONException { + matomoRequestBuilder = new MatomoRequestBuilder(); + String json = matomoRequestBuilder.buildJSON(new MatomoBulkRequest("my-token", null)); + JSONObject jsonObject = new JSONObject(json); + assertThat(jsonObject.getString("token_auth"), is("my-token")); + + assertThat(jsonObject.has("requests"), is(false)); + } + + + @Test + public void testSinglelRequest() throws JSONException { + matomoRequestBuilder = new MatomoRequestBuilder(); + MatomoRequestDetails e1 = + new MatomoRequestDetails() + .addParameter("rec", "1") + .addParameter("idsite", "1") + .addParameter("action_name", "my-action"); + String json = matomoRequestBuilder.buildJSON(new MatomoBulkRequest("my-token", List.of(e1))); + JSONObject jsonObject = new JSONObject(json); + assertThat(jsonObject.getString("token_auth"), is("my-token")); + + assertThat(jsonObject.has("requests"), is(true)); + + JSONArray jsonArray = jsonObject.getJSONArray("requests"); + assertThat(jsonArray.length(), is(1)); + + String requestUrl = jsonArray.getString(0); + assertThat(requestUrl, startsWith("?")); + + String[] parameters = requestUrl.substring(1).split("&"); + assertThat( + List.of(parameters), + containsInAnyOrder("rec=1", "idsite=1", "action_name=my-action") + ); + } + + @Test + public void testMultipleRequests() throws JSONException { + matomoRequestBuilder = new MatomoRequestBuilder(); + MatomoRequestDetails e1 = + new MatomoRequestDetails() + .addParameter("rec", "1") + .addParameter("idsite", "1") + .addParameter("action_name", "my-first-action"); + MatomoRequestDetails e2 = + new MatomoRequestDetails() + .addParameter("rec", "1") + .addParameter("idsite", "1") + .addParameter("action_name", "my-second-action"); + String json = matomoRequestBuilder.buildJSON(new MatomoBulkRequest("my-token", List.of(e1,e2))); + JSONObject jsonObject = new JSONObject(json); + assertThat(jsonObject.getString("token_auth"), is("my-token")); + + assertThat(jsonObject.has("requests"), is(true)); + + JSONArray jsonArray = jsonObject.getJSONArray("requests"); + assertThat(jsonArray.length(), is(2)); + + + List list = jsonArray.toList(); + assertThat(list.size(), is(2)); + assertThat(list, + containsInAnyOrder( + new UrlParameterMatcher("rec=1", "idsite=1", "action_name=my-second-action"), + new UrlParameterMatcher("rec=1", "idsite=1", "action_name=my-first-action") + ) + ); + } + + private static final class UrlParameterMatcher extends BaseMatcher { + + List parameterList; + + public UrlParameterMatcher(String... parameters) { + parameterList = List.of(parameters); + } + + @Override + public boolean matches(Object o) { + if (!(o instanceof String s)) { + return false; + } + return parameterList.stream().allMatch(s::contains); + } + + @Override + public void describeTo(Description description) { + description.appendText("an UrlParameterMatcher with the following params: ") + .appendValue(parameterList); + } + } + +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/client/MatomoResponseReaderTest.java b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoResponseReaderTest.java new file mode 100644 index 000000000000..1f8bd33df740 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/client/MatomoResponseReaderTest.java @@ -0,0 +1,77 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.client; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +import org.dspace.AbstractUnitTest; +import org.junit.Before; +import org.junit.Test; + +public class MatomoResponseReaderTest extends AbstractUnitTest { + + MatomoResponseReader matomoResponseReader; + + @Before + public void setUp() throws Exception { + matomoResponseReader = new MatomoResponseReader(); + } + + @Test + public void testReadNullResponse() { + assertThat(matomoResponseReader.fromJSON(null), nullValue()); + } + + @Test + public void testReadEmptyResponse() { + MatomoResponse actual = matomoResponseReader.fromJSON(""); + assertThat(actual, nullValue()); + } + + @Test + public void testReadEmptyJsonResponse() { + MatomoResponse actual = matomoResponseReader.fromJSON("{}"); + assertThat(actual, notNullValue()); + assertThat(actual.status(), nullValue()); + assertThat(actual.tracked(), is(0)); + assertThat(actual.invalid(), is(0)); + assertThat(actual.invalidIndices(), nullValue()); + } + + @Test + public void testReadSuccessResponse() { + MatomoResponse actual = + matomoResponseReader.fromJSON( + "{\"status\":\"success\",\"tracked\":1,\"invalid\":0,\"invalid_indices\":[]}" + ); + assertThat(actual, notNullValue()); + assertThat(actual.status(), is("success")); + assertThat(actual.tracked(), is(1)); + assertThat(actual.invalid(), is(0)); + assertThat(actual.invalidIndices(), notNullValue()); + assertThat(actual.invalidIndices().length, is(0)); + } + + @Test + public void testReadFailedResponse() { + MatomoResponse actual = + matomoResponseReader.fromJSON( + "{\"status\":\"success\",\"tracked\":0,\"invalid\":1,\"invalid_indices\":[0]}" + ); + assertThat(actual, notNullValue()); + assertThat(actual.status(), is("success")); + assertThat(actual.tracked(), is(0)); + assertThat(actual.invalid(), is(1)); + assertThat(actual.invalidIndices(), notNullValue()); + assertThat(actual.invalidIndices().length, is(1)); + assertThat(actual.invalidIndices()[0], is(0)); + } +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/model/MatomoCookieConverterTest.java b/dspace-api/src/test/java/org/dspace/matomo/model/MatomoCookieConverterTest.java new file mode 100644 index 000000000000..e908209946d1 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/model/MatomoCookieConverterTest.java @@ -0,0 +1,55 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import static org.junit.Assert.assertEquals; + +import java.util.List; + +import org.dspace.AbstractUnitTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class MatomoCookieConverterTest extends AbstractUnitTest { + + @Test + public void testSingleCookieConverter() { + String convert = MatomoCookieConverter.convert( + new MatomoRequestDetails() + .addCookie("cookie1", "value1") + ); + assertEquals("cookie1=value1", convert); + } + + @Test + public void testMultipleCookieConverter() { + String convert = MatomoCookieConverter.convert( + new MatomoRequestDetails() + .addCookie("cookie1", "value1") + .addCookie("cookie2", "value2") + ); + assertEquals("cookie1=value1;cookie2=value2", convert); + } + + @Test + public void testMultipleDetailsConverter() { + String convert = MatomoCookieConverter.convert( + List.of( + new MatomoRequestDetails() + .addCookie("cookie1", "value1") + .addCookie("cookie2", "value2"), + new MatomoRequestDetails() + .addCookie("cookie1", "value11") + .addCookie("cookie2", "value22") + ) + ); + assertEquals("cookie1=value11;cookie2=value22", convert); + } +} diff --git a/dspace-api/src/test/java/org/dspace/matomo/model/MatomoRequestDetailsBuilderTest.java b/dspace-api/src/test/java/org/dspace/matomo/model/MatomoRequestDetailsBuilderTest.java new file mode 100644 index 000000000000..c01f4df28c6f --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/model/MatomoRequestDetailsBuilderTest.java @@ -0,0 +1,647 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; + +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import org.dspace.AbstractUnitTest; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamService; +import org.dspace.content.service.DSpaceObjectService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.matomo.factory.MatomoRequestCookieIdentifierEnricher; +import org.dspace.matomo.factory.MatomoRequestCookieSessionEnricher; +import org.dspace.matomo.factory.MatomoRequestCountryEnricher; +import org.dspace.matomo.factory.MatomoRequestCustomCookiesEnricher; +import org.dspace.matomo.factory.MatomoRequestCustomVariablesEnricher; +import org.dspace.matomo.factory.MatomoRequestDetailsEnricher; +import org.dspace.matomo.factory.MatomoRequestDetailsEnricherFactory; +import org.dspace.matomo.factory.MatomoRequestIpAddressEnricher; +import org.dspace.matomo.factory.MatomoRequestTrackerIdentifierParamEnricher; +import org.dspace.service.ClientInfoService; +import org.dspace.usage.UsageEvent; +import org.hamcrest.CoreMatchers; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +public class MatomoRequestDetailsBuilderTest extends AbstractUnitTest { + + MatomoRequestDetailsBuilder builder; + List enrichers; + + @Mock + UsageEvent usageEvent; + @Mock + HttpServletRequest request; + @Mock + Context context; + + final String siteId = "test"; + + @Before + public void setUp() throws Exception { + enrichers = new ArrayList<>(); + builder = new MatomoRequestDetailsBuilder(enrichers, siteId); + } + + @Test + public void testDefaultBuilders() { + MatomoRequestDetails request = builder.build(usageEvent); + assertThat(request.parameters, CoreMatchers.notNullValue()); + assertThat( + request.parameters, + Matchers.hasEntry( + Matchers.is("idsite"), + Matchers.is(siteId) + ) + ); + assertThat( + request.parameters, + Matchers.hasEntry( + Matchers.is("rec"), + Matchers.is("1") + ) + ); + } + + @Test + public void testActionNameBuilder() throws SQLException { + enrichers.add(MatomoRequestDetailsEnricherFactory.actionNameEnricher()); + + Item item = Mockito.mock(Item.class); + Mockito.when(item.getName()).thenReturn("item-name"); + Mockito.when(item.getType()).thenReturn(Constants.ITEM); + Mockito.when(this.usageEvent.getObject()).thenReturn(item); + Mockito.when(this.usageEvent.getContext()).thenReturn(context); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("action_name"), + Matchers.is("item-name") + ) + ); + + + Bitstream bitstream = Mockito.mock(Bitstream.class); + Mockito.when(bitstream.getType()).thenReturn(Constants.BITSTREAM); + Mockito.when(this.usageEvent.getObject()).thenReturn(bitstream); + + try (MockedStatic mock = Mockito.mockStatic(ContentServiceFactory.class)) { + ContentServiceFactory serviceFactory = Mockito.mock(ContentServiceFactory.class); + Mockito.when(ContentServiceFactory.getInstance()).thenReturn(serviceFactory); + DSpaceObjectService bitstreamService = Mockito.mock(BitstreamService.class); + Mockito.when(serviceFactory.getDSpaceObjectService(bitstream)) + .thenReturn(bitstreamService); + Mockito.when(bitstreamService.getParentObject(context, bitstream)) + .thenReturn(item); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("action_name"), + Matchers.is("item-name") + ) + ); + } + + } + + @Test + public void testUserAgentEnricher() { + enrichers.add(MatomoRequestDetailsEnricherFactory.userAgentEnricher()); + Mockito.when(request.getHeader(Mockito.eq("USER-AGENT"))).thenReturn("custom-agent"); + Mockito.when(usageEvent.getRequest()).thenReturn(request); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("ua"), + Matchers.is("custom-agent") + ) + ); + } + + @Test + public void testUrlEnricher() { + enrichers.add(MatomoRequestDetailsEnricherFactory.urlEnricher()); + + Item item = Mockito.mock(Item.class); + Mockito.when(item.getType()).thenReturn(Constants.ITEM); + UUID itemUUID = UUID.randomUUID(); + Mockito.when(item.getID()).thenReturn(itemUUID); + Mockito.when(this.usageEvent.getObject()).thenReturn(item); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("url"), + Matchers.containsString("/items/" + itemUUID) + ) + ); + + UUID bitstreamUUID = UUID.randomUUID(); + Bitstream bitstream = Mockito.mock(Bitstream.class); + Mockito.when(bitstream.getID()).thenReturn(bitstreamUUID); + Mockito.when(this.usageEvent.getObject()).thenReturn(bitstream); + + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("url"), + Matchers.containsString("/bitstreams/" + bitstreamUUID) + ) + ); + + UUID bundleUUID = UUID.randomUUID(); + DSpaceObject object = Mockito.mock(Bundle.class); + Mockito.when(object.getType()).thenReturn(Constants.BUNDLE); + Mockito.when(usageEvent.getObject()).thenReturn(object); + + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("url"), + Matchers.emptyString() + ) + ); + + Mockito.when(usageEvent.getObject()).thenReturn(null); + + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("url"), + Matchers.emptyString() + ) + ); + + } + + @Test + public void testMatomoRequestCountryEnricher() { + MatomoRequestCountryEnricher countryEnricher = new MatomoRequestCountryEnricher(); + enrichers.add(countryEnricher); + + String country = Locale.ITALIAN.getCountry(); + Mockito.when(request.getLocale()).thenReturn(Locale.ITALIAN); + Mockito.when(usageEvent.getRequest()).thenReturn(request); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("country"), + Matchers.is(country) + ) + ); + + Mockito.when(request.getLocale()).thenReturn(null); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("country"), + Matchers.emptyString() + ) + ); + + Mockito.when(usageEvent.getRequest()).thenReturn(null); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("country"), + Matchers.emptyString() + ) + ); + } + + @Test + public void testMatomoIpEnricher() { + ClientInfoService clientInfo = Mockito.mock(ClientInfoService.class); + MatomoRequestIpAddressEnricher ipAddressEnricher = new MatomoRequestIpAddressEnricher(clientInfo); + enrichers.add(ipAddressEnricher); + + Mockito.when(usageEvent.getRequest()).thenReturn(request); + Mockito.when(clientInfo.getClientIp(request)).thenReturn("fake-ip"); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("cip"), + Matchers.is("fake-ip") + ) + ); + + Mockito.when(clientInfo.getClientIp(request)).thenReturn(null); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("cip"), + Matchers.emptyString() + ) + ); + + Mockito.when(clientInfo.getClientIp(request)).thenReturn(""); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("cip"), + Matchers.emptyString() + ) + ); + } + + @Test + public void testDownloadEnricher() { + enrichers.add(MatomoRequestDetailsEnricherFactory.downloadEnricher()); + + Item item = Mockito.mock(Item.class); + Mockito.when(item.getType()).thenReturn(Constants.ITEM); + Mockito.when(this.usageEvent.getObject()).thenReturn(item); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("download"), + Matchers.emptyString() + ) + ); + + UUID bitstreamUUID = UUID.randomUUID(); + Bitstream bitstream = Mockito.mock(Bitstream.class); + Mockito.when(bitstream.getID()).thenReturn(bitstreamUUID); + Mockito.when(this.usageEvent.getObject()).thenReturn(bitstream); + + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("download"), + Matchers.containsString("/bitstreams/" + bitstreamUUID + "/download") + ) + ); + + Mockito.when(usageEvent.getObject()).thenReturn(null); + + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("download"), + Matchers.emptyString() + ) + ); + + } + + @Test + public void testMatomoCookieIdentifierEnricher() { + MatomoRequestCookieIdentifierEnricher cookieEnricher = new MatomoRequestCookieIdentifierEnricher(); + enrichers.add(cookieEnricher); + + Cookie cookie = Mockito.mock(Cookie.class); + Mockito.when(cookie.getName()).thenReturn("_pk_id.1.1fff"); + Mockito.when(cookie.getValue()).thenReturn("3225aebdb98b13f9.1740076196."); + + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { cookie }); + Mockito.when(usageEvent.getRequest()).thenReturn(request); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("_id"), + Matchers.is("3225aebdb98b13f9") + ) + ); + + Mockito.when(request.getCookies()).thenReturn(null); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.not( + Matchers.hasEntry( + Matchers.is("_id"), + Matchers.any(String.class) + ) + ) + ); + + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { }); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.not( + Matchers.hasEntry( + Matchers.is("_id"), + Matchers.any(String.class) + ) + ) + ); + + Mockito.when(cookie.getName()).thenReturn("_pk_id.1.1fff"); + Mockito.when(cookie.getValue()).thenReturn("wrongvalue.1.2"); + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { cookie }); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.not( + Matchers.hasEntry( + Matchers.is("_id"), + Matchers.any(String.class) + ) + ) + ); + + Mockito.when(cookie.getName()).thenReturn("_pk_id.1.1fff"); + Mockito.when(cookie.getValue()).thenReturn(""); + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { cookie }); + requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.not( + Matchers.hasEntry( + Matchers.is("_id"), + Matchers.any(String.class) + ) + ) + ); + } + + @Test + public void testMatomoCustomCookieEnricher() { + MatomoRequestCustomCookiesEnricher cookiesEnricher = + new MatomoRequestCustomCookiesEnricher("_pk_ref,_pk_hsr,_pk_ses"); + enrichers.add(cookiesEnricher); + + Cookie pkRefCookie = Mockito.mock(Cookie.class); + Mockito.when(pkRefCookie.getName()).thenReturn("_pk_ref.1.1fff"); + Mockito.when(pkRefCookie.getValue()).thenReturn("http://localhost/home"); + + Cookie pkHsr = Mockito.mock(Cookie.class); + Mockito.when(pkHsr.getName()).thenReturn("_pk_hsr.1.1fff"); + Mockito.when(pkHsr.getValue()).thenReturn("hsr-value"); + + Cookie pkSes = Mockito.mock(Cookie.class); + Mockito.when(pkSes.getName()).thenReturn("_pk_ses.1.1fff"); + Mockito.when(pkSes.getValue()).thenReturn("1"); + + Cookie noCustom = Mockito.mock(Cookie.class); + Mockito.when(noCustom.getName()).thenReturn("_pk_custom.1.1fff"); + + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { pkRefCookie, pkHsr, pkSes, noCustom }); + Mockito.when(usageEvent.getRequest()).thenReturn(request); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.cookies, + Matchers.allOf( + Matchers.hasEntry( + Matchers.is("_pk_ref.1.1fff"), + Matchers.is("http://localhost/home") + ), + Matchers.hasEntry( + Matchers.is("_pk_hsr.1.1fff"), + Matchers.is("hsr-value") + ), + Matchers.hasEntry( + Matchers.is("_pk_ses.1.1fff"), + Matchers.is("1") + ) + ) + ); + + assertThat( + requestDetails.cookies, + Matchers.not( + Matchers.hasEntry( + Matchers.is("_pk_custom.1.1fff"), + Matchers.any(String.class) + ) + ) + ); + } + + @Test + public void testMatomoCookieSessionEnricher() { + MatomoRequestCookieSessionEnricher sessionEnricher = new MatomoRequestCookieSessionEnricher(); + enrichers.add(sessionEnricher); + + Cookie sessionCookie = Mockito.mock(Cookie.class); + Mockito.when(sessionCookie.getName()).thenReturn("MATOMO_SESSID"); + Mockito.when(sessionCookie.getValue()).thenReturn("44d4405e1652daa7a7e451c019cf01db"); + + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { sessionCookie }); + Mockito.when(usageEvent.getRequest()).thenReturn(request); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.cookies, + Matchers.hasEntry( + Matchers.is("MATOMO_SESSID"), + Matchers.is("44d4405e1652daa7a7e451c019cf01db") + ) + ); + } + + @Test + public void testMatomoCustomVaribalesEnricher() { + MatomoRequestCustomVariablesEnricher customVariablesEnricher = new MatomoRequestCustomVariablesEnricher(); + enrichers.add(customVariablesEnricher); + + Cookie cvar = Mockito.mock(Cookie.class); + Mockito.when(cvar.getName()).thenReturn("_pk_cvar.1.1fff"); + Mockito.when(cvar.getValue()).thenReturn("{\"1\":[\"key1\",\"value1\"],\"2\":[\"key2\",\"value2\"]}"); + + Mockito.when(request.getCookies()).thenReturn(new Cookie[] { cvar }); + Mockito.when(usageEvent.getRequest()).thenReturn(request); + + MatomoRequestDetails requestDetails = builder.build(usageEvent); + assertThat( + requestDetails.parameters, + Matchers.hasEntry( + Matchers.is("_cvar"), + Matchers.is("{\"1\":[\"key1\",\"value1\"],\"2\":[\"key2\",\"value2\"]}") + ) + ); + } + + /** + * Test the enrich method with an empty parameter map. + * This tests the edge case where the request's parameter map is empty, which is implicitly handled in the method. + */ + @Test + public void testEnrichWithEmptyParameterMap() { + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + UsageEvent usageEvent = mock(UsageEvent.class); + HttpServletRequest request = mock(HttpServletRequest.class); + when(usageEvent.getRequest()).thenReturn(request); + when(usageEvent.getRequest().getParameterMap()).thenReturn(new HashMap<>()); + + MatomoRequestDetails result = enricher.enrich(usageEvent, matomoRequestDetails); + + assertEquals(matomoRequestDetails, result); + } + + /** + * Test the enrich method with an invalid tracker ID. + * This tests the edge case where the tracker ID is present but does not match the expected format. + */ + @Test + public void testEnrichWithInvalidTrackerId() { + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + UsageEvent usageEvent = mock(UsageEvent.class); + when(usageEvent.getRequest()).thenReturn(mock(HttpServletRequest.class)); + + Map parameterMap = new HashMap<>(); + parameterMap.put("trackerId", new String[] {"invalidTrackerID"}); + when(usageEvent.getRequest().getParameterMap()).thenReturn(parameterMap); + + MatomoRequestDetails result = enricher.enrich(usageEvent, matomoRequestDetails); + + assertEquals(matomoRequestDetails, result); + } + + /** + * Test the enrich method with a UsageEvent that has a null request. + * This tests the edge case where the UsageEvent's request is null, which is explicitly handled in the method. + */ + @Test + public void testEnrichWithNullRequest() { + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + UsageEvent usageEvent = mock(UsageEvent.class); + when(usageEvent.getRequest()).thenReturn(null); + + MatomoRequestDetails result = enricher.enrich(usageEvent, matomoRequestDetails); + + assertEquals(matomoRequestDetails, result); + } + + /** + * Test the enrich method with a null UsageEvent. + * This tests the edge case where the UsageEvent is null, which is explicitly handled in the method. + */ + @Test + public void testEnrichWithNullUsageEvent() { + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + + MatomoRequestDetails result = enricher.enrich(null, matomoRequestDetails); + + assertEquals(matomoRequestDetails, result); + } + + /** + * Test case for the enrich method when the UsageEvent is null. + * This test verifies that the method returns the original MatomoRequestDetails + * object without modifications when the input UsageEvent is null. + */ + @Test + public void test_enrich_whenUsageEventIsNull() { + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + MatomoRequestDetails matomoRequestDetails = mock(MatomoRequestDetails.class); + + MatomoRequestDetails result = enricher.enrich(null, matomoRequestDetails); + + assertEquals(matomoRequestDetails, result); + } + + /** + * Test case for enrich method when UsageEvent and HttpServletRequest are not null, + * but the parameter map does not contain a valid tracker identifier. + * Expected: The original MatomoRequestDetails should be returned unchanged. + */ + @Test + public void test_enrich_withInvalidParameter() { + // Arrange + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + UsageEvent usageEvent = mock(UsageEvent.class); + HttpServletRequest request = mock(HttpServletRequest.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + + Map parameterMap = new HashMap<>(); + parameterMap.put("trackerId", new String[] {"invalidValue"}); + + when(usageEvent.getRequest()).thenReturn(request); + when(request.getParameterMap()).thenReturn(parameterMap); + + // Act + MatomoRequestDetails result = enricher.enrich(usageEvent, matomoRequestDetails); + + // Assert + assertEquals(matomoRequestDetails, result); + } + + /** + * Tests the enrich method when a valid tracker identifier is present in the request parameters. + * This test verifies that the method adds the tracker identifier to the MatomoRequestDetails + * when the usage event contains a valid tracker ID in its request parameters. + */ + @Test + public void test_enrich_with_valid_tracker_id() { + // Arrange + MatomoRequestTrackerIdentifierParamEnricher enricher = new MatomoRequestTrackerIdentifierParamEnricher(); + UsageEvent usageEvent = mock(UsageEvent.class); + HttpServletRequest request = mock(HttpServletRequest.class); + MatomoRequestDetails matomoRequestDetails = new MatomoRequestDetails(); + + Map parameterMap = new HashMap<>(); + parameterMap.put("trackerId", new String[] {"1234567890abcdef"}); + + when(usageEvent.getRequest()).thenReturn(request); + when(request.getParameterMap()).thenReturn(parameterMap); + + MatomoRequestDetails result = enricher.enrich(usageEvent, matomoRequestDetails); + + assertThat( + result.parameters, + Matchers.hasEntry( + Matchers.is("_id"), + Matchers.is("1234567890abcdef") + ) + ); + } + +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/matomo/model/MatomoRequestDetailsSplitterTest.java b/dspace-api/src/test/java/org/dspace/matomo/model/MatomoRequestDetailsSplitterTest.java new file mode 100644 index 000000000000..74880ad5849c --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/matomo/model/MatomoRequestDetailsSplitterTest.java @@ -0,0 +1,150 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.matomo.model; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; + +import java.util.List; +import java.util.Map; + +import org.dspace.AbstractUnitTest; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; +import org.junit.Test; + +public class MatomoRequestDetailsSplitterTest extends AbstractUnitTest { + + @Test + public void testMatomoRequestDetailsSingleRequest() { + Map> split = MatomoRequestDetailsSplitter.split( + List.of( + new MatomoRequestDetails() + .addParameter("_id", "first") + .addParameter("param1", "value1") + ) + ); + MatcherAssert.assertThat( + split.keySet(), + CoreMatchers.hasItem("first") + ); + MatcherAssert.assertThat( + split.get("first").get(0).parameters, + CoreMatchers.allOf( + hasEntry("_id", "first"), hasEntry("param1", "value1") + ) + ); + } + + @Test + public void testMatomoRequestDetailsNoIdRequest() { + Map> split = MatomoRequestDetailsSplitter.split( + List.of( + new MatomoRequestDetails() + .addParameter("param1", "value1") + .addParameter("param2", "value2") + ) + ); + MatcherAssert.assertThat( + split.keySet(), + CoreMatchers.hasItem("default") + ); + MatcherAssert.assertThat( + split.get("default").get(0).parameters, + CoreMatchers.allOf( + hasEntry("param2", "value2"), hasEntry("param1", "value1") + ) + ); + } + + + @Test + public void testMatomoMultipleRequests() { + Map> split = MatomoRequestDetailsSplitter.split( + List.of( + new MatomoRequestDetails() + .addParameter("_id", "first") + .addParameter("param2", "value2"), + new MatomoRequestDetails() + .addParameter("_id", "first") + .addParameter("param1", "value1") + ) + ); + MatcherAssert.assertThat( + split.keySet(), + hasItem("first") + ); + MatcherAssert.assertThat( + split.keySet(), + not(hasItem("default")) + ); + MatcherAssert.assertThat( + split.get("first").get(0).parameters, + allOf( + hasEntry("param2", "value2"), + hasEntry("_id", "first") + ) + ); + MatcherAssert.assertThat( + split.get("first").get(1).parameters, + allOf( + hasEntry("param1", "value1"), + hasEntry("_id", "first") + ) + ); + } + + @Test + public void testMatomoMultipleRequestsNoId() { + Map> split = MatomoRequestDetailsSplitter.split( + List.of( + new MatomoRequestDetails() + .addParameter("_id", "first") + .addParameter("param2", "value2"), + new MatomoRequestDetails() + .addParameter("_id", "first") + .addParameter("param1", "value1"), + new MatomoRequestDetails() + .addParameter("param3", "value3") + .addParameter("param4", "value4") + ) + ); + MatcherAssert.assertThat( + split.keySet(), + hasItem("first") + ); + MatcherAssert.assertThat( + split.keySet(), + hasItem("default") + ); + MatcherAssert.assertThat( + split.get("first").get(0).parameters, + allOf( + hasEntry("param2", "value2"), + hasEntry("_id", "first") + ) + ); + MatcherAssert.assertThat( + split.get("first").get(1).parameters, + allOf( + hasEntry("param1", "value1"), + hasEntry("_id", "first") + ) + ); + MatcherAssert.assertThat( + split.get("default").get(0).parameters, + allOf( + hasEntry("param3", "value3"), + hasEntry("param4", "value4") + ) + ); + } + +} diff --git a/dspace-api/src/test/java/org/dspace/storage/bitstore/BitstreamStorageServiceImplIT.java b/dspace-api/src/test/java/org/dspace/storage/bitstore/BitstreamStorageServiceImplIT.java new file mode 100644 index 000000000000..ca3ac768a353 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/storage/bitstore/BitstreamStorageServiceImplIT.java @@ -0,0 +1,269 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.storage.bitstore; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.authorize.AuthorizeException; +import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Bitstream; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Context; +import org.dspace.storage.bitstore.factory.StorageServiceFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * UMD Customization + * + * This class was provided to DSpace in Pull Request 10940 + * This comment can be removed once this application has been upgraded to a + * DSpace version containing the pull request. + */ +public class BitstreamStorageServiceImplIT extends AbstractIntegrationTestWithDatabase { + private BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService(); + private BitstreamStorageServiceImpl bitstreamStorageService = + (BitstreamStorageServiceImpl) StorageServiceFactory.getInstance().getBitstreamStorageService(); + private Collection collection; + + private Map originalBitstores; + + private static final Integer SOURCE_STORE = 0; + private static final Integer DEST_STORE = 1; + + @Rule + public final TemporaryFolder tempStoreDir = new TemporaryFolder(); + + @Before + public void setup() throws Exception { + + context.turnOffAuthorisationSystem(); + + parentCommunity = CommunityBuilder.createCommunity(context) + .build(); + + collection = CollectionBuilder.createCollection(context, parentCommunity) + .build(); + + originalBitstores = bitstreamStorageService.getStores(); + Map stores = new HashMap<>(); + DSBitStoreService sourceStore = new DSBitStoreService(); + sourceStore.setBaseDir(tempStoreDir.newFolder("src")); + + stores.put(SOURCE_STORE, sourceStore); + bitstreamStorageService.setStores(stores); + + context.restoreAuthSystemState(); + } + + @After + public void cleanUp() throws IOException { + // Restore the bitstore storage stores + bitstreamStorageService.setStores(originalBitstores); + } + + /** + * Test batch commit checkpointing, using the default batch commit size of 1 + * + * @throws Exception if an exception occurs. + */ + @Test + public void testDefaultBatchCommitSize() throws Exception { + Context context = this.context; + + // Destination assetstore fails after two bitstreams have been migrated + DSBitStoreService destinationStore = new LimitedTempDSBitStoreService(tempStoreDir, 2); + Map stores = bitstreamStorageService.getStores(); + stores.put(DEST_STORE, destinationStore); + + // Create three bitstreams in the source assetstore + createBitstreams(context, 3); + + // Three bitstreams in source assetstore at the start + assertThat(bitstreamService.countByStoreNumber(context, SOURCE_STORE).intValue(), equalTo(3)); + + // No bitstreams in destination assetstore at the start + assertThat(bitstreamService.countByStoreNumber(context, DEST_STORE).intValue(), equalTo(0)); + + /// Commit any pending transaction to database + context.commit(); + + // Migrate bitstreams + context.turnOffAuthorisationSystem(); + + boolean deleteOld = false; + Integer batchCommitSize = 1; + try { + bitstreamStorageService.migrate( + context, SOURCE_STORE, DEST_STORE, deleteOld, + batchCommitSize + ); + fail("IOException should have been thrown"); + } catch (IOException ioe) { + // Rollback any pending transaction + context.rollback(); + } + + context.restoreAuthSystemState(); + + // One bitstream should still be in the source assetstore, due to the + // interrupted migration + assertThat(bitstreamService.countByStoreNumber(context, SOURCE_STORE).intValue(), equalTo(1)); + + // Two bitstreams should have migrated to the destination assetstore + assertThat(bitstreamService.countByStoreNumber(context, DEST_STORE).intValue(), equalTo(2)); + } + + /** + * Test batch commit checkpointing, using the default batch commit size of 3 + * + * @throws Exception if an exception occurs. + */ + @Test + public void testBatchCommitSizeThree() throws Exception { + Context context = this.context; + + // Destination assetstore fails after four bitstreams have been migrated + DSBitStoreService destinationStore = new LimitedTempDSBitStoreService(tempStoreDir, 4); + Map stores = bitstreamStorageService.getStores(); + stores.put(DEST_STORE, destinationStore); + + // Create five bitstreams in the source assetstore + createBitstreams(context, 5); + + // Five bitstreams in source assetstore at the start + assertThat(bitstreamService.countByStoreNumber(context, SOURCE_STORE).intValue(), equalTo(5)); + + // No bitstreams in destination assetstore at the start + assertThat(bitstreamService.countByStoreNumber(context, DEST_STORE).intValue(), equalTo(0)); + + // Commit any pending transaction to database + context.commit(); + + // Migrate bitstreams + context.turnOffAuthorisationSystem(); + + boolean deleteOld = false; + Integer batchCommitSize = 3; + try { + bitstreamStorageService.migrate( + context, SOURCE_STORE, DEST_STORE, deleteOld, + batchCommitSize + ); + fail("IOException should have been thrown"); + } catch (IOException ioe) { + // Rollback any pending transaction + context.rollback(); + } + + context.restoreAuthSystemState(); + + // Since the batch commit size is 3, only three bitstreams should be + // marked as migrated, so there should still be two bitstreams + // in the source assetstore, due to the interrupted migration + assertThat(bitstreamService.countByStoreNumber(context, SOURCE_STORE).intValue(), equalTo(2)); + + // Three bitstreams should have migrated to the destination assetstore + assertThat(bitstreamService.countByStoreNumber(context, DEST_STORE).intValue(), equalTo(3)); + } + + private void createBitstreams(Context context, int numBitstreams) + throws SQLException { + context.turnOffAuthorisationSystem(); + for (int i = 0; i < numBitstreams; i++) { + String content = "Test bitstream " + i; + createBitstream(content); + } + context.restoreAuthSystemState(); + context.commit(); + } + + private Bitstream createBitstream(String content) { + try { + return BitstreamBuilder + .createBitstream(context, createItem(), toInputStream(content)) + .build(); + } catch (SQLException | AuthorizeException | IOException e) { + throw new RuntimeException(e); + } + } + + private Item createItem() { + return ItemBuilder.createItem(context, collection) + .withTitle("Test item") + .build(); + } + + + private InputStream toInputStream(String content) { + return IOUtils.toInputStream(content, UTF_8); + } + + + /** + * DSBitStoreService variation that only allows a limited number of puts + * to the bit store before throwing an IOException, to test the + * error handling of the BitstreamStorageService.migrate() method. + */ + class LimitedTempDSBitStoreService extends DSBitStoreService { + // The number of put calls allowed before throwing an IOException + protected int maxPuts = Integer.MAX_VALUE; + + // The number of "put" method class seen so far. + protected int putCallCount = 0; + + /** + * Constructor. + * + * @param maxPuts the number of put calls to allow before throwing an + * IOException + */ + public LimitedTempDSBitStoreService(TemporaryFolder tempStoreDir, int maxPuts) throws IOException { + super(); + setBaseDir(tempStoreDir.newFolder()); + this.maxPuts = maxPuts; + } + + /** + * Store a stream of bits. + * + * After "maxPut" number of calls, this method throws an IOException. + * @param in The stream of bits to store + * @throws java.io.IOException If a problem occurs while storing the bits + */ + @Override + public void put(Bitstream bitstream, InputStream in) throws IOException { + putCallCount = putCallCount + 1; + if (putCallCount > maxPuts) { + throw new IOException("Max 'put' method calls exceeded"); + } else { + super.put(bitstream, in); + } + } + } +} diff --git a/dspace-iiif/pom.xml b/dspace-iiif/pom.xml index d1a71e664e65..9df0dfe6d336 100644 --- a/dspace-iiif/pom.xml +++ b/dspace-iiif/pom.xml @@ -15,7 +15,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT .. diff --git a/dspace-oai/pom.xml b/dspace-oai/pom.xml index 813c1f594b27..4e0fc44e1bcf 100644 --- a/dspace-oai/pom.xml +++ b/dspace-oai/pom.xml @@ -8,7 +8,7 @@ dspace-parent org.dspace - 8.1 + 8.1-drum-1-SNAPSHOT .. diff --git a/dspace-rdf/pom.xml b/dspace-rdf/pom.xml index aeaecf8a9849..34f4fa4dcd05 100644 --- a/dspace-rdf/pom.xml +++ b/dspace-rdf/pom.xml @@ -9,7 +9,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT .. diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index edc34eafcf92..4001dbe9355e 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -14,7 +14,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT .. @@ -330,7 +330,7 @@ spring-boot-starter-aop ${spring-boot.version} - + org.springframework.boot spring-boot-starter-actuator diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/CASLoginFilter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/CASLoginFilter.java new file mode 100644 index 000000000000..91550279a10a --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/CASLoginFilter.java @@ -0,0 +1,139 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.security; + +import java.io.IOException; +import java.util.ArrayList; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.core.Utils; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; + +/** + * UMD-custom class for handling CAS authentication. This class is based on + * the org.dspace.app.rest.security.ShibbolethLoginFilter class. + * + * This class is in the "dspace-server-webapp" module, because there does not + * appear to be a Spring mechanism for overriding the + * org.dspace.app.rest.security.WebSecurityConfiguration` class in which the + * login filter classes are configured. + * + * See "dspace/docs/CASAuthentication.md" for more information about CAS + * authentication. + */ +public class CASLoginFilter extends StatelessLoginFilter { + private static final Logger log = LogManager.getLogger(CASLoginFilter.class); + + private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + + public CASLoginFilter(String url, AuthenticationManager authenticationManager, + RestAuthenticationService restAuthenticationService) { + super(url, "GET", authenticationManager, restAuthenticationService); + logger.info("Created CASLoginFilter"); + } + + @Override + public Authentication attemptAuthentication(HttpServletRequest req, + HttpServletResponse res) throws AuthenticationException { + // Commenting out this check, because otherwise the CASAuthentication + // class would need to be in this module (or a parent module), instead + // of in the "additions" module. + /* + if (!CASAuthentication.isEnabled()) { + throw new ProviderNotFoundException("CAS is disabled."); + } + */ + + // In the case of CAS, this method does NOT actually authenticate us + // (the authentication has already happened CAS). So, this call to + // "authenticate()" is just triggering/ CASAuthentication.authenticate() + // to check for a valid CAS login, and if found, the current user + // is considered authenticated via CAS. + // + // NOTE: because this authentication is implicit, we pass in an empty DSpaceAuthentication + return authenticationManager.authenticate(new DSpaceAuthentication()); + } + + @Override + protected void successfulAuthentication(HttpServletRequest req, + HttpServletResponse res, + FilterChain chain, + Authentication auth) throws IOException, ServletException { + // Once we've gotten here, we know we have a successful login + // (i.e. attemptAuthentication() succeeded) + + // This method is using the same logic/mechanisms as + // org.dspace.app.rest.security.ShibbolethLoginFilter for handling + // the JWT. + + DSpaceAuthentication dSpaceAuthentication = (DSpaceAuthentication) auth; + log.debug("CAS authentication successful for EPerson {}. Sending back temporary auth cookie", + dSpaceAuthentication.getName()); + + // OVERRIDE DEFAULT behavior of StatelessLoginFilter to return a temporary authentication cookie containing + // the Auth Token (JWT). This Cookie is required because we *redirect* the user back to the client/UI after + // a successful Shibboleth login. Headers cannot be sent via a redirect, so a Cookie must be sent to provide + // the auth token to the client. On the next request from the client, the cookie is read and destroyed & the + // Auth token is only used in the Header from that point forward. + restAuthenticationService.addAuthenticationDataForUser(req, res, dSpaceAuthentication, true); + + // redirect user after completing CAS authentication, sending along the temporary auth cookie + redirectAfterSuccess(req, res); + } + + + /** + * After successful login, redirect to the DSpace URL specified by this CAS + * request (in the "redirectUrl" request parameter). If that 'redirectUrl' + * is not valid or trusted for this DSpace site, then return a 400 error. + * + * @param request + * @param response + * @throws IOException + */ + private void redirectAfterSuccess(HttpServletRequest request, HttpServletResponse response) throws IOException { + // This method is using the same logic/mechanisms as + // org.dspace.app.rest.security.ShibbolethLoginFilter + String redirectUrl = request.getParameter("redirectUrl"); + + // If redirectUrl unspecified, default to the configured UI + if (StringUtils.isEmpty(redirectUrl)) { + redirectUrl = configurationService.getProperty("dspace.ui.url"); + } + + // Validate that the redirectURL matches either the server or UI hostname. It *cannot* be an arbitrary URL. + String redirectHostName = Utils.getHostName(redirectUrl); + String serverHostName = Utils.getHostName(configurationService.getProperty("dspace.server.url")); + ArrayList allowedHostNames = new ArrayList<>(); + allowedHostNames.add(serverHostName); + String[] allowedUrls = configurationService.getArrayProperty("rest.cors.allowed-origins"); + for (String url : allowedUrls) { + allowedHostNames.add(Utils.getHostName(url)); + } + + if (StringUtils.equalsAnyIgnoreCase(redirectHostName, allowedHostNames.toArray(new String[0]))) { + log.debug("CAS redirecting to " + redirectUrl); + response.sendRedirect(redirectUrl); + } else { + log.error("Invalid CAS redirectURL=" + redirectUrl + + ". URL doesn't match hostname of server or UI!"); + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + "Invalid redirectURL! Must match server or ui hostname."); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/WebSecurityConfiguration.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/WebSecurityConfiguration.java index af7116a2bea5..333263c088ac 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/WebSecurityConfiguration.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/WebSecurityConfiguration.java @@ -149,6 +149,15 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { .addFilterBefore(new ShibbolethLoginFilter("/api/authn/shibboleth", HttpMethod.GET.name(), authenticationManager, restAuthenticationService), LogoutFilter.class) + + // UMD Customization + // Add a filter before our CAS endpoints to do authentication + // based on the data in the HTTP request. + .addFilterBefore(new CASLoginFilter("/api/authn/cas", authenticationManager(), + restAuthenticationService), + LogoutFilter.class) + // End UMD Customization + // Add a filter before our ORCID endpoints to do the authentication based on the data in the HTTP request. // This endpoint only responds to GET as the actual authentication is performed by ORCID, which then // redirects to this endpoint to forward the authentication data to DSpace. diff --git a/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/api/discovery.xml b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/api/discovery.xml new file mode 100644 index 000000000000..92917818132e --- /dev/null +++ b/dspace-server-webapp/src/test/data/dspaceFolder/config/spring/api/discovery.xml @@ -0,0 +1,3404 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.rights + + + + + + + + + + + + + + + dc.rights + + + + + + + + dc.description.provenance + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (search.resourcetype:Item AND latestVersion:true) OR search.resourcetype:Collection OR search.resourcetype:Community + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.title + dc.contributor.author + dc.creator + dc.subject + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item OR search.resourcetype:Collection OR search.resourcetype:Community + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.title + dc.contributor.author + dc.creator + dc.subject + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true + + withdrawn:true OR discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.title + dc.contributor.author + dc.creator + dc.subject + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.title + dc.contributor.author + dc.creator + dc.subject + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (search.resourcetype:Item AND latestVersion:true) OR search.resourcetype:WorkspaceItem OR search.resourcetype:XmlWorkflowItem + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:WorkspaceItem AND supervised:true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:PoolTask OR search.resourcetype:ClaimedTask + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:XmlWorkflowItem + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:WorkspaceItem OR search.resourcetype:XmlWorkflowItem + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:Publication + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:Publication + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:Person + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:Person + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:Project + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:Project + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:OrgUnit + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:OrgUnit + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:JournalIssue + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:JournalIssue + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:JournalVolume + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:JournalVolume + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:Journal + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND entityType_keyword:Journal + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND (entityType_keyword:OrgUnit OR entityType_keyword:Person) + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item AND latestVersion:true AND entityType_keyword:OrgUnit AND dc.type:FundingOrganization + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:Item + search.entitytype:${researcher-profile.entity-type:Person} + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:LDNMessageEntity + + + + + + + + + + + + + + + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_INCOMING} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_INCOMING} + in_reply_to:* + activity_stream_type_keyword:Accept + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_PROCESSED} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_INCOMING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_PROCESSED} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_INCOMING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_FAILED} OR queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_UNMAPPED_ACTION} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_INCOMING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_UNTRUSTED} + + + + + + + + + + search.resourcetype:Item + {!join from=relateditem_authority to=search.resourceid fromIndex=${solr.multicorePrefix}search}notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_INCOMING} + -withdrawn:true AND -discoverable:false + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_OUTGOING} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_OUTGOING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_PROCESSED} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_OUTGOING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_QUEUED} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_OUTGOING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_QUEUED_FOR_RETRY} + + + + + + + + + + search.resourcetype:LDNMessageEntity + notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_OUTGOING} + queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_FAILED} OR queue_status_authority:#{T(org.dspace.app.ldn.LDNMessageEntity).QUEUE_STATUS_UNMAPPED_ACTION} + + + + + + + + + + search.resourcetype:Item + {!join from=relateditem_authority to=search.resourceid fromIndex=${solr.multicorePrefix}search}notification_type_keyword:#{T(org.dspace.app.ldn.LDNMessageEntity).TYPE_OUTGOING} + -withdrawn:true AND -discoverable:false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.title + + + + + + + + + + + relation.isAuthorOfPublication.latestForDiscovery + + + + + + + + + + + relation.isProjectOfPublication.latestForDiscovery + + + + + + + + + + + + relation.isOrgUnitOfPublication.latestForDiscovery + + + + + + + + + + + relation.isPublicationOfJournalIssue.latestForDiscovery + + + + + + + + + + + relation.isJournalOfPublication.latestForDiscovery + + + + + + + + + + + dc.contributor.author + dc.creator + + + + + + + + + + + + + + + dspace.entity.type + + + + + + + + + + + + + + dc.subject.* + + + + + + + + + + + + + + dc.date.issued + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dc.type + + + + + + + + + dc.identifier + + + + + + + + + placeholder.placeholder.placeholder + + + + + + + + + + placeholder.placeholder.placeholder + + + + + + + + + person.jobTitle + + + + + + + + + + + + + + + person.knowsLanguage + + + + + + + + + + + + + person.birthDate + + + + + + + + + + + + + + + + + person.familyName + + + + + + + + + + + person.givenName + + + + + + + + + + + relation.isOrgUnitOfPerson.latestForDiscovery + + + + + + + + + + + relation.isProjectOfPerson.latestForDiscovery + + + + + + + + + + + relation.isPublicationOfAuthor.latestForDiscovery + + + + + + + + + + + + organization.address.addressCountry + + + + + + + + + + + + + + + organization.address.addressLocality + + + + + + + + + + + + + + + organization.foundingDate + + + + + + + + + + + + + + + + organization.legalName + + + + + + + + + + + relation.isPersonOfOrgUnit.latestForDiscovery + + + + + + + + + + + relation.isProjectOfOrgUnit.latestForDiscovery + + + + + + + + + + + relation.isPublicationOfOrgUnit.latestForDiscovery + + + + + + + + + + + creativework.keywords + + + + + + + + + + + + + + + creativework.datePublished + + + + + + + + + + + + + + + + publicationissue.issueNumber + + + + + + + + + + + relation.isPublicationOfJournalIssue.latestForDiscovery + + + + + + + + + + + publicationVolume.volumeNumber + + + + + + + + + + + relation.isIssueOfJournalVolume.latestForDiscovery + + + + + + + + + + + relation.isJournalOfVolume.latestForDiscovery + + + + + + + + + + + creativework.publisher + + + + + + + + + + + + + + + creativework.editor + + + + + + + + + + + + + + + relation.isVolumeOfJournal.latestForDiscovery + + + + + + + + + + + + + + placeholder.placeholder.placeholder + + + + + + + + + + relation.isOrgUnitOfProject.latestForDiscovery + + + + + + + + + + + + relation.isPersonOfProject.latestForDiscovery + + + + + + + + + + + + relation.isPublicationOfProject.latestForDiscovery + + + + + + + + + + + relation.isContributorOfPublication.latestForDiscovery + + + + + + + + + + + relation.isPublicationOfContributor.latestForDiscovery + + + + + + + + + + + relation.isFundingAgencyOfProject.latestForDiscovery + + + + + + + + + + + relation.isProjectOfFundingAgency.latestForDiscovery + + + + + + + + + + + + + placeholder.placeholder.placeholder + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + datacite.relation.isReferencedBy + datacite.relation.isSupplementedBy + + + + + + + + + + + + + + coar.notify.endorsedBy + + + + + + + + + + + + + + datacite.relation.isReviewedBy + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyRestRepositoryIT.java index 30890d7ef838..6ccd424c2402 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyRestRepositoryIT.java @@ -32,6 +32,7 @@ import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -39,6 +40,9 @@ * This class handles all Authority related IT. It alters some config to run the tests, but it gets cleared again * after every test */ +// UMD Customization +@Ignore("UMD - These tests consistently fail when run locally, but pass on Jenkins, so possibly an Apple Silicon issue") +// End UMD Customization public class VocabularyRestRepositoryIT extends AbstractControllerIntegrationTest { @Autowired diff --git a/dspace-services/pom.xml b/dspace-services/pom.xml index 0b3c3f33da38..dcbac319a796 100644 --- a/dspace-services/pom.xml +++ b/dspace-services/pom.xml @@ -9,7 +9,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT diff --git a/dspace-sword/pom.xml b/dspace-sword/pom.xml index 0487692a8202..7a733d7e1d20 100644 --- a/dspace-sword/pom.xml +++ b/dspace-sword/pom.xml @@ -15,7 +15,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT .. diff --git a/dspace-swordv2/pom.xml b/dspace-swordv2/pom.xml index 64754b1995ca..28f47f648958 100644 --- a/dspace-swordv2/pom.xml +++ b/dspace-swordv2/pom.xml @@ -13,7 +13,7 @@ org.dspace dspace-parent - 8.1 + 8.1-drum-1-SNAPSHOT .. diff --git a/dspace/bin/dspace b/dspace/bin/dspace index 24644aae9112..a4094540a384 100644 --- a/dspace/bin/dspace +++ b/dspace/bin/dspace @@ -14,6 +14,23 @@ BINDIR=`dirname $0` DSPACEDIR=`cd "$BINDIR/.." ; pwd` +# UMD Customization +# Ensure "additions" jar is place before other DSpace jars in the classpath +# (but after any specified by the CLASSPATH environment variable, so that it +# can override any stock DSpace classes. This issue may be corrected +# in future DSpace versions, see: +# * https://github.com/DSpace/DSpace/issues/3248 and +# * https://github.com/DSpace/DSpace/issues/8506 +ADDITIONS_JAR=`ls $DSPACEDIR/lib/additions-*.jar 2>/dev/null` +if [ "$ADDITIONS_JAR" != "" ]; then + if [ "$CLASSPATH" = "" ]; then + CLASSPATH=$ADDITIONS_JAR + else + CLASSPATH=$CLASSPATH:$ADDITIONS_JAR + fi +fi +# End UMD Customization + # Get the JARs in $DSPACEDIR/lib JARS="$DSPACEDIR/lib/*" @@ -39,7 +56,17 @@ if [ "$JAVA_OPTS" = "" ]; then JAVA_OPTS="-Xmx256m -Dfile.encoding=UTF-8" fi -export JAVA_OPTS="$JAVA_OPTS -Dlog4j2.configurationFile=$DSPACEDIR/config/log4j2-cli.xml" +# UMD Customization +# Enable the dspace-cli log file to be overridden by a "UMD_DSPACE_CLI_LOG_CONFIG" +# environment variable. This allows scripts such as "load-etd" to specify their +# own log files. +if [ "$UMD_DSPACE_CLI_LOG_CONFIG" = "" ]; then + # Default to using log4j2-cli.xml + UMD_DSPACE_CLI_LOG_CONFIG="-Dlog4j2.configurationFile=$DSPACEDIR/config/log4j2-cli.xml" +fi + +export JAVA_OPTS="$JAVA_OPTS $UMD_DSPACE_CLI_LOG_CONFIG" +# End UMD Customization # Now invoke Java java $JAVA_OPTS \ diff --git a/dspace/bin/load-etd b/dspace/bin/load-etd new file mode 100755 index 000000000000..02034cc99934 --- /dev/null +++ b/dspace/bin/load-etd @@ -0,0 +1,101 @@ +#!/usr/bin/perl + +##################################################################### +# Copyright 2006, The University of Maryland. All rights reserved. +# +# Program: load-etd +# +# Author: Ben Wallberg +# +# Purpose: Load ProQuest ETD into DSpace. +# +# Usage: See PrintUsageAndExit() +# +# Comments: Adapted from load-diss +# +##################################################################### + +use Getopt::Std; + +# +# Check the command-line parameters +# +GetCmdLine(); + +# Get directories +chomp($bindir = `dirname $0`); + +# Use UMD_DSPACE_CLI_LOG_CONFIG to override the dspace-cli log configuration +$ENV{UMD_DSPACE_CLI_LOG_CONFIG} = "-Dlog4j2.configurationFile=log4j2-etdloader.xml"; +$ENV{JAVA_OPTS} .= " -Ddspace.log.init.disable=true"; +$ENV{JAVA_OPTS} .= " -Detdloader.zipfile=$zipfile"; +$ENV{JAVA_OPTS} .= " -Detdloader.singleitem=$item" if (defined $item); + +# Add additions jar to front of classpath to ensure the overlay files +# has precedence over stock implementation +chomp($bindir_realpath = `realpath $bindir`); +chomp($dspacedir = `dirname $bindir_realpath`); +chomp($additions_jar = `find $dspacedir/lib/ -type f -name "additions*.jar"`); +$prev_classpath = $ENV{CLASSPATH}; +$classpath_separator = ":"; +$ENV{CLASSPATH} = $additions_jar; +$ENV{CLASSPATH} .= $classpath_separator.$prev_classpath if ($prev_classpath ne ""); + +@cmd = ("$bindir/dspace", "dsrun", "edu.umd.lib.dspace.app.EtdLoader"); + +#print $ENV{JAVA_OPTS}; +#print (join ' ',@cmd) . "\n"; +system(@cmd); + +exit 0; + + +########################################################## GetCmdLine +# Process the command line options. +##################################################################### + +sub GetCmdLine { + + if ($#ARGV < 0 || ($#ARGV == 0 && $ARGV[0] eq "-h")) { + PrintUsageAndExit(); + } + + if ( ! getopts('i:r:')) { + print "Error in option processing\n\n"; + PrintUsageAndExit(); + } + + if (! defined $opt_i) { + PrintUsageAndExit("Error: is a required parameter"); + } + $zipfile = $opt_i; + + if (defined $opt_r) { + $item = $opt_r; + } + +} + + +################################################### PrintUsageAndExit +# Print the program usage and exit. +##################################################################### + +sub PrintUsageAndExit { + + if (@_) { + foreach $msg (@_) { + print "$msg\n"; + } + print "\n"; + } + + print <<'EOS'; +Usage: load-etd -i [-r ] + : zip file containing items to load + : process a single item, not the entire zip file +EOS + + exit 1; +} + diff --git a/dspace/bin/load-etd-nightly b/dspace/bin/load-etd-nightly new file mode 100755 index 000000000000..b9f21ed1f707 --- /dev/null +++ b/dspace/bin/load-etd-nightly @@ -0,0 +1,53 @@ +#!/bin/csh + +# load-etd-nightly +# +# Check for new upload files in the incoming directory and load +# them into dspace. +# + +# Get command-line arguments +if ($#argv != 1) then + echo "Usage: $0 " + echo " : Directory containing Proquest ETD files" + exit 0 +endif + +set datadir = `cd $argv[1]; pwd` +set bindir = `dirname $0` +set incomingdir = $datadir/incoming +set processeddir = $datadir/processed + +# Check for incoming files +ls $incomingdir/etdadmin_upload_*.zip >& /dev/null +if ($status == 0) then + echo Files found in $incomingdir + + @ count = 0 + + foreach upload ($incomingdir/etdadmin_upload_*.zip) + set zipfile = `basename $upload` + + # Load the files from this archive + echo + echo ====================================================================== + echo Loading archive file: $incomingdir/$zipfile + $bindir/load-etd -i $incomingdir/$zipfile + + # Move archive to the processed directory + if (! -d $processeddir ) then + mkdir -p $processeddir + endif + echo + echo ' ' Moving $zipfile to $processeddir + mv $incomingdir/$zipfile $processeddir + + @ count = $count + 1 + end +endif + + + + + + diff --git a/dspace/bin/mail b/dspace/bin/mail new file mode 100755 index 000000000000..f9876eb8e3d9 --- /dev/null +++ b/dspace/bin/mail @@ -0,0 +1,27 @@ +#!/usr/bin/perl + +# only send mail if there's anything on stdin + +use Config::Properties; +open my $fh, '<', '/dspace/config/local.cfg' + or die "unable to open configuration file"; +my $properties = Config::Properties->new(); +$properties->load($fh); +$host = $properties->getProperty('dspace.hostname'); +$from_address = "cron@" . $host; +$smtp_host = $properties->getProperty('mail.server'); +$smtp_port = $properties->getProperty('mail.server.port'); + +$mail=0; +while() { + if (! $mail) { + $mail = 1; + @args = ('-r', $from_address, '-S', "mta=smtp://$smtp_host:$smtp_port", '-S', 'v15-compat', '-S', 'smtp-auth=none'); + push(@args, @ARGV); + open(PROC, "|-", "/usr/bin/s-nail", @args) || die; + } + print PROC; +} +if ($mail) { + close PROC; +} diff --git a/dspace/bin/script-mail-wrapper b/dspace/bin/script-mail-wrapper new file mode 100755 index 000000000000..4d467f49a755 --- /dev/null +++ b/dspace/bin/script-mail-wrapper @@ -0,0 +1,74 @@ +#!/bin/bash + +# script-mail-wrapper +# +# Calls the wrapped script, and passes the output to the Splunk log +# and to the "mail" script. +# +# script-mail-wrapper