diff --git a/.github/actions/setup-maven/action.yml b/.github/actions/setup-maven/action.yml index 4cf09f34231..e3e0de47329 100644 --- a/.github/actions/setup-maven/action.yml +++ b/.github/actions/setup-maven/action.yml @@ -14,7 +14,7 @@ runs: using: composite steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ inputs.git-reference }} - name: Determine Java version by reading the Maven property diff --git a/.github/workflows/check_property_files.yml b/.github/workflows/check_property_files.yml index 505310aab35..4ae2b8aee5e 100644 --- a/.github/workflows/check_property_files.yml +++ b/.github/workflows/check_property_files.yml @@ -9,7 +9,7 @@ jobs: name: Duplicate Keys runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Run duplicates detection script shell: bash run: tests/check_duplicate_properties.sh @@ -18,7 +18,7 @@ jobs: name: Metadata Blocks Properties runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup GraalVM + Native Image uses: graalvm/setup-graalvm@v1 with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9f4e94b9f5b..105469139ec 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -61,7 +61,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index a4c52805156..a5dddc755d1 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -20,7 +20,7 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: # Checkout the pull request code as when merged - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - uses: actions/setup-java@v4 diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 8abe33bdefc..142363cbe1a 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -173,7 +173,7 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - name: Setup Trivy binary for vulnerability scanning - uses: aquasecurity/setup-trivy@v0.2.3 + uses: aquasecurity/setup-trivy@v0.2.4 with: version: v0.63.0 @@ -199,7 +199,7 @@ jobs: - configbaker-image steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 ### BASE IMAGE - name: Render README for base image @@ -272,6 +272,6 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/base + repository: gdcc/configbaker short-description: "Dataverse Config Baker Container Image providing setup tooling and more" readme-filepath: ./modules/container-configbaker/README.md diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index f0afb01dd18..0e060113ba0 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-java@v4 with: @@ -47,10 +47,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Download war artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: built-app path: ./ diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index a3b5882626c..b41606d0c99 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -10,7 +10,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - id: lookup run: | echo "sphinx_version=$(grep Sphinx== ./doc/sphinx-guides/requirements.txt | tr -s "=" | cut -f 2 -d=)" | tee -a "${GITHUB_OUTPUT}" diff --git a/.github/workflows/maven_cache_management.yml b/.github/workflows/maven_cache_management.yml index fedf63b7c54..f266b804534 100644 --- a/.github/workflows/maven_cache_management.yml +++ b/.github/workflows/maven_cache_management.yml @@ -32,7 +32,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Determine Java version from Parent POM run: echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} - name: Set up JDK ${{ env.JAVA_VERSION }} @@ -80,7 +80,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Cleanup caches run: | gh extension install actions/gh-actions-cache diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index efefdaee02a..a416d5323f0 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -37,7 +37,7 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v4 with: @@ -103,7 +103,7 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v4 with: @@ -112,7 +112,7 @@ jobs: cache: maven # Get the build output from the unit test job - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: name: java-artifacts - run: | @@ -137,7 +137,7 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-java@v4 with: java-version: '17' @@ -145,7 +145,7 @@ jobs: cache: maven # Get the build output from the integration test job - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: name: java-reportdir - run: tar -xvf java-reportdir.tar diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml index 804b04f696a..fb91f2c718e 100644 --- a/.github/workflows/reviewdog_checkstyle.yml +++ b/.github/workflows/reviewdog_checkstyle.yml @@ -10,7 +10,7 @@ jobs: name: Checkstyle job steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run check style uses: nikitasavinov/checkstyle-action@master with: diff --git a/.github/workflows/scripts/containers/maintain-application.sh b/.github/workflows/scripts/containers/maintain-application.sh index b68c2a53d96..e34c609b0ad 100755 --- a/.github/workflows/scripts/containers/maintain-application.sh +++ b/.github/workflows/scripts/containers/maintain-application.sh @@ -126,15 +126,15 @@ for BRANCH in "$@"; do # 6. Let's put together what tags we want added to this build run TAG_OPTIONS="" if ! (( IS_DEV )); then - TAG_OPTIONS="-Dapp.image=$APP_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + TAG_OPTIONS="-Dapp.image=$APP_IMAGE_REF -Dapp.image.tag.1=$NEXT_REV_TAG" # In case of the current release, add the "latest" tag as well. if (( IS_CURRENT_RELEASE )); then - TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + TAG_OPTIONS="$TAG_OPTIONS -Dapp.image.tag.2=latest" fi else # shellcheck disable=SC2016 UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout ) - TAG_OPTIONS="-Ddocker.tags.upcoming=$UPCOMING_TAG" + TAG_OPTIONS="-Dapp.image.tag.1=$UPCOMING_TAG" # For the dev branch we only have rolling tags and can add them now already SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\"]") @@ -148,9 +148,8 @@ for BRANCH in "$@"; do # Build the application image, but skip the configbaker image (that's a different job)! # shellcheck disable=SC2046 mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ - -Dconf.skipBuild -Dbase.image="${BASE_IMAGE_REF}" \ - -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ - $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) + -Dconf.skipBuild -Dbase.image="${BASE_IMAGE_REF}" $TAG_OPTIONS \ + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push"; fi ) else echo "Skipping Maven build as requested by DRY_RUN=1" fi diff --git a/.github/workflows/scripts/containers/maintain-base.sh b/.github/workflows/scripts/containers/maintain-base.sh index 5b9ae738b98..ace76d7f3cd 100755 --- a/.github/workflows/scripts/containers/maintain-base.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -120,6 +120,13 @@ for BRANCH in "$@"; do CURRENT_REV_TAG="${BASE_IMAGE_REF#*:}-r$REV" NEXT_REV_TAG="${BASE_IMAGE_REF#*:}-r$(( REV + 1 ))" + # 6a. Determine if we must newly release an "r0" because a rolling tag from development is now released. + IS_NEW_RELEASE=0 + if ! (( IS_DEV )) && [ "$REV" = "-1" ]; then + echo "This is a newly released version of Dataverse - forcing build, as no r0 image is present on Docker Hub" + IS_NEW_RELEASE=1 + fi + # 7. Let's put together what tags we want added to this build run TAG_OPTIONS="" if ! (( IS_DEV )); then @@ -139,12 +146,12 @@ for BRANCH in "$@"; do # 8. Let's build the base image if necessary NEWER_IMAGE=0 - if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then + if (( NEWER_JAVA_IMAGE + NEWER_PKGS + IS_NEW_RELEASE + FORCE_BUILD > 0 )); then if ! (( DRY_RUN )); then # shellcheck disable=SC2046 mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ - $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push"; fi ) else echo "Skipping Maven build as requested by DRY_RUN=1" fi diff --git a/.github/workflows/scripts/containers/maintain-configbaker.sh b/.github/workflows/scripts/containers/maintain-configbaker.sh index 0b5b60b459c..2cd89efaeb6 100755 --- a/.github/workflows/scripts/containers/maintain-configbaker.sh +++ b/.github/workflows/scripts/containers/maintain-configbaker.sh @@ -122,15 +122,15 @@ for BRANCH in "$@"; do # 6. Let's put together what tags we want added to this build run TAG_OPTIONS="" if ! (( IS_DEV )); then - TAG_OPTIONS="-Dconf.image=$CONFIG_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + TAG_OPTIONS="-Dconf.image=$CONFIG_IMAGE_REF -Dconf.image.tag.1=$NEXT_REV_TAG" # In case of the current release, add the "latest" tag as well. if (( IS_CURRENT_RELEASE )); then - TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + TAG_OPTIONS="$TAG_OPTIONS -Dconf.image.tag.2=latest" fi else # shellcheck disable=SC2016 UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f . -Dexpression=conf.image.tag -Dconf.image.tag='${app.image.version}-${conf.image.flavor}' -q -DforceStdout ) - TAG_OPTIONS="-Ddocker.tags.upcoming=$UPCOMING_TAG" + TAG_OPTIONS="-Dconf.image.tag.1=$UPCOMING_TAG" # For the dev branch we only have rolling tags and can add them now already SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\"]") @@ -145,9 +145,8 @@ for BRANCH in "$@"; do # shellcheck disable=SC2046 mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Dapp.skipBuild -Dconf.image.base="${BASE_IMAGE_REF}" \ - -Dmaven.main.skip -Dmaven.test.skip -Dmaven.war.skip \ - -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ - $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) + -Dmaven.main.skip -Dmaven.test.skip -Dmaven.war.skip $TAG_OPTIONS \ + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push"; fi ) else echo "Skipping Maven build as requested by DRY_RUN=1" fi diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index fb9cf5a0a1f..55a760bad21 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: shellcheck uses: reviewdog/action-shellcheck@v1 with: diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml index cc09992edac..d49d399b792 100644 --- a/.github/workflows/shellspec.yml +++ b/.github/workflows/shellspec.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Run Shellspec run: | cd tests/shell @@ -30,7 +30,7 @@ jobs: container: image: rockylinux/rockylinux:9 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install shellspec run: | curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share @@ -47,7 +47,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Run Shellspec run: | cd tests/shell diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 6398edca412..9dc722c5992 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-java@v4 with: java-version: '17' @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-java@v4 with: java-version: '17' diff --git a/conf/keycloak/Dockerfile b/conf/keycloak/Dockerfile index 76088f402c7..d2c85ee3335 100644 --- a/conf/keycloak/Dockerfile +++ b/conf/keycloak/Dockerfile @@ -14,10 +14,10 @@ RUN mvn clean package # ------------------------------------------ # Stage 2: Build Keycloak Image # ------------------------------------------ -FROM quay.io/keycloak/keycloak:26.1.4 +FROM quay.io/keycloak/keycloak:26.3.2 # Add the Oracle JDBC jars -ARG ORACLE_JDBC_VERSION=23.7.0.25.01 +ARG ORACLE_JDBC_VERSION=23.8.0.25.04 ADD --chown=keycloak:keycloak https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc11/${ORACLE_JDBC_VERSION}/ojdbc11-${ORACLE_JDBC_VERSION}.jar /opt/keycloak/providers/ojdbc11.jar ADD --chown=keycloak:keycloak https://repo1.maven.org/maven2/com/oracle/database/nls/orai18n/${ORACLE_JDBC_VERSION}/orai18n-${ORACLE_JDBC_VERSION}.jar /opt/keycloak/providers/orai18n.jar @@ -29,7 +29,7 @@ COPY --from=builder /app/target/keycloak-dv-builtin-users-authenticator-1.0-SNAP # Copy additional configurations COPY ./builtin-users-spi/conf/quarkus.properties /opt/keycloak/conf/ -COPY ./test-realm.json /opt/keycloak/data/import/ +COPY ./test-realm-include-spi.json /opt/keycloak/data/import/ # Set the Keycloak command ENTRYPOINT ["/opt/keycloak/bin/kc.sh"] diff --git a/conf/keycloak/builtin-users-spi/pom.xml b/conf/keycloak/builtin-users-spi/pom.xml index afb3495c2be..36cf6548d01 100644 --- a/conf/keycloak/builtin-users-spi/pom.xml +++ b/conf/keycloak/builtin-users-spi/pom.xml @@ -100,7 +100,7 @@ - 26.1.4 + 26.3.2 17 3.2.0 0.4 diff --git a/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/adapters/DataverseUserAdapter.java b/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/adapters/DataverseUserAdapter.java index d9609fe1c1f..a47eeee1749 100644 --- a/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/adapters/DataverseUserAdapter.java +++ b/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/adapters/DataverseUserAdapter.java @@ -1,6 +1,7 @@ package edu.harvard.iq.keycloak.auth.spi.adapters; import edu.harvard.iq.keycloak.auth.spi.models.DataverseUser; +import edu.harvard.iq.keycloak.auth.spi.providers.DataverseUserStorageProviderFactory; import org.keycloak.component.ComponentModel; import org.keycloak.models.GroupModel; import org.keycloak.models.KeycloakSession; @@ -15,10 +16,13 @@ public class DataverseUserAdapter extends AbstractUserAdapterFederatedStorage { protected DataverseUser dataverseUser; protected String keycloakId; + private static final String ATTRIBUTE_NAME_IDP = "idp"; + public DataverseUserAdapter(KeycloakSession session, RealmModel realm, ComponentModel model, DataverseUser dataverseUser) { super(session, realm, model); this.dataverseUser = dataverseUser; keycloakId = StorageId.keycloakId(model, dataverseUser.getBuiltinUser().getId().toString()); + this.setSingleAttribute(ATTRIBUTE_NAME_IDP, DataverseUserStorageProviderFactory.PROVIDER_ID); } @Override diff --git a/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java b/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java index c8fab3d0b7b..5256d5df683 100644 --- a/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java +++ b/conf/keycloak/builtin-users-spi/src/main/java/edu/harvard/iq/keycloak/auth/spi/services/DataverseUserService.java @@ -27,11 +27,16 @@ public DataverseUser getUserById(String id) { DataverseBuiltinUser builtinUser = em.find(DataverseBuiltinUser.class, persistenceId); if (builtinUser == null) { - logger.debugf("User not found for external ID: %s", persistenceId); + logger.debugf("Builtin user not found for external ID: %s", persistenceId); return null; } - DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(builtinUser.getUsername()); + String username = builtinUser.getUsername(); + DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(username); + if (authenticatedUser == null) { + logger.debugf("Authenticated user not found by username: %s", username); + return null; + } return new DataverseUser(authenticatedUser, builtinUser); } @@ -43,11 +48,15 @@ public DataverseUser getUserByUsername(String username) { .getResultList(); if (users.isEmpty()) { - logger.debugf("User not found by username: %s", username); + logger.debugf("Builtin user not found by username: %s", username); return null; } DataverseAuthenticatedUser authenticatedUser = getAuthenticatedUserByUsername(username); + if (authenticatedUser == null) { + logger.debugf("Authenticated user not found by username: %s", username); + return null; + } return new DataverseUser(authenticatedUser, users.get(0)); } @@ -59,7 +68,7 @@ public DataverseUser getUserByEmail(String email) { .getResultList(); if (authUsers.isEmpty()) { - logger.debugf("User not found by email: %s", email); + logger.debugf("Authenticated user not found by email: %s", email); return null; } @@ -68,6 +77,11 @@ public DataverseUser getUserByEmail(String email) { .setParameter("username", username) .getResultList(); + if (builtinUsers.isEmpty()) { + logger.debugf("Builtin user not found by username: %s", username); + return null; + } + return new DataverseUser(authUsers.get(0), builtinUsers.get(0)); } diff --git a/conf/keycloak/docker-compose.yml b/conf/keycloak/docker-compose.yml index 272d8ace363..4b5cf80daca 100644 --- a/conf/keycloak/docker-compose.yml +++ b/conf/keycloak/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.9" services: keycloak: - image: 'quay.io/keycloak/keycloak:26.1.4' + image: 'quay.io/keycloak/keycloak:26.3.2' command: - "start-dev" - "--import-realm" diff --git a/conf/keycloak/run-keycloak.sh b/conf/keycloak/run-keycloak.sh index 9f851a558c7..9ea34a99416 100755 --- a/conf/keycloak/run-keycloak.sh +++ b/conf/keycloak/run-keycloak.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -DOCKER_IMAGE="quay.io/keycloak/keycloak:26.1.4" +DOCKER_IMAGE="quay.io/keycloak/keycloak:26.3.2" KEYCLOAK_USER="kcadmin" KEYCLOAK_PASSWORD="kcpassword" KEYCLOAK_PORT=8090 diff --git a/conf/keycloak/test-realm-include-spi.json b/conf/keycloak/test-realm-include-spi.json new file mode 100644 index 00000000000..3ab6ac2477e --- /dev/null +++ b/conf/keycloak/test-realm-include-spi.json @@ -0,0 +1,2383 @@ +{ + "id": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "realm": "test", + "displayName": "", + "displayNameHtml": "", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 1800, + "ssoSessionMaxLifespan": 36000, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "none", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxTemporaryLockouts": 0, + "bruteForceStrategy": "MULTIPLE", + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "075daee1-5ab2-44b5-adbf-fa49a3da8305", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + }, + { + "id": "b4ff9091-ddf9-4536-b175-8cfa3e331d71", + "name": "default-roles-test", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + }, + { + "id": "e6d31555-6be6-4dee-bc6a-40a53108e4c2", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "1955bd12-5f86-4a74-b130-d68a8ef6f0ee", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "1109c350-9ab1-426c-9876-ef67d4310f35", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "980c3fd3-1ae3-4b8f-9a00-d764c939035f", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "5363e601-0f9d-4633-a8c8-28cb0f859b7b", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "59aa7992-ad78-48db-868a-25d6e1d7db50", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "impersonation", + "view-authorization", + "query-users", + "query-groups", + "manage-clients", + "manage-realm", + "view-identity-providers", + "query-realms", + "manage-authorization", + "manage-identity-providers", + "manage-users", + "view-users", + "view-realm", + "create-client", + "view-clients", + "manage-events", + "query-clients", + "view-events" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "112f53c2-897d-4c01-81db-b8dc10c5b995", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "c7f57bbd-ef32-4a64-9888-7b8abd90777a", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "8885dac8-0af3-45af-94ce-eff5e801bb80", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2673346c-b0ef-4e01-8a90-be03866093af", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "b7182885-9e57-445f-8dae-17c16eb31b5d", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "ba7bfe0c-cb07-4a47-b92c-b8132b57e181", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "13a8f0fc-647d-4bfe-b525-73956898e550", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "ef4c57dc-78c2-4f9a-8d2b-0e97d46fc842", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2875da34-006c-4b7f-bfc8-9ae8e46af3a2", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "c8c8f7dc-876b-4263-806f-3329f7cd5fd3", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "21b84f90-5a9a-4845-a7ba-bbd98ac0fcc4", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "6fd64c94-d663-4501-ad77-0dcf8887d434", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "b321927a-023c-4d2a-99ad-24baf7ff6d83", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2fc21160-78de-457b-8594-e5c76cde1d5e", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + } + ], + "test": [], + "security-admin-console": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "07ee59b5-dca6-48fb-83d4-2994ef02850e", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "b57d62bb-77ff-42bd-b8ff-381c7288f327", + "attributes": {} + } + ], + "account": [ + { + "id": "17d2f811-7bdf-4c73-83b4-1037001797b8", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "f4999a71-d4c3-4d7c-afb0-ba522e457bd3", + "name": "view-groups", + "description": "${role_view-groups}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "d1ff44f9-419e-42fd-98e8-1add1169a972", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "14c23a18-ae2d-43c9-b0c0-aaf6e0c7f5b0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "6fbe58af-d2fe-4d66-95fe-a2e8a818cb55", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "bdfd02bc-6f6a-47d2-82bc-0ca52d78ff48", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "782f3b0c-a17b-4a87-988b-1a711401f3b0", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "8a3bfe15-66d9-4f3d-83ac-801d682d42b0", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + } + ] + } + }, + "groups": [ + { + "id": "d46f94c2-3b47-4288-b937-9cf918e54f0a", + "name": "admins", + "path": "/admins", + "subGroups": [], + "attributes": {}, + "realmRoles": [], + "clientRoles": {} + }, + { + "id": "e992ce15-baac-48a0-8834-06f6fcf6c05b", + "name": "curators", + "path": "/curators", + "subGroups": [], + "attributes": {}, + "realmRoles": [], + "clientRoles": {} + }, + { + "id": "531cf81d-a700-4336-808f-37a49709b48c", + "name": "members", + "path": "/members", + "subGroups": [], + "attributes": {}, + "realmRoles": [], + "clientRoles": {} + } + ], + "defaultRole": { + "id": "b4ff9091-ddf9-4536-b175-8cfa3e331d71", + "name": "default-roles-test", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpPolicyCodeReusable": false, + "otpSupportedApplications": [ + "totpAppFreeOTPName", + "totpAppGoogleName", + "totpAppMicrosoftAuthenticatorName" + ], + "localizationTexts": {}, + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyExtraOrigins": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "webAuthnPolicyPasswordlessExtraOrigins": [], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account", + "view-groups" + ] + } + ] + }, + "clients": [ + { + "id": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/test/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/test/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "realm_client": "false", + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "5d99f721-027c-478d-867d-61114e0a8192", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/test/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/test/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "realm_client": "false", + "post.logout.redirect.uris": "+", + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "e181a0ce-9a04-4468-a38a-aaef9f78f989", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "5eccc178-121e-4d0f-bcb2-04ae3c2e52ed", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "realm_client": "false", + "client.use.lightweight.access.token.enabled": "true", + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "b57d62bb-77ff-42bd-b8ff-381c7288f327", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "realm_client": "true", + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "dada0ae8-ee9f-415a-9685-42da7c563660", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "realm_client": "true", + "post.logout.redirect.uris": "+" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "bf7cf550-3875-4f97-9878-b2419a854058", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/test/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/test/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "realm_client": "false", + "client.use.lightweight.access.token.enabled": "true", + "post.logout.redirect.uris": "+", + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "ff845e16-e200-4894-ab51-37d8b9f2a445", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "9c27faa8-4b8d-4ad9-9cd1-880032ef06aa", + "clientId": "test", + "name": "A Test Client", + "description": "Use for hacking and testing away a confidential client", + "rootUrl": "", + "adminUrl": "", + "baseUrl": "", + "surrogateAuthRequired": false, + "enabled": true, + "secret": "94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8", + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": true, + "protocol": "openid-connect", + "attributes": { + "realm_client": "false", + "oidc.ciba.grant.enabled": "false", + "client.secret.creation.time": "1684735831", + "backchannel.logout.session.required": "true", + "post.logout.redirect.uris": "+", + "display.on.consent.screen": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "protocolMappers": [ + { + "id": "480de80c-656e-401c-b146-f777db45340b", + "name": "idp", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "userinfo.token.claim": "true", + "user.attribute": "idp", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "idp", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "acr", + "roles", + "profile", + "basic", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "72f29e57-92fa-437b-828c-2b9d6fe56192", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "consent.screen.text": "${addressScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "59581aea-70d6-4ee8-bec2-1fea5fc497ae", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "0f4cc9ba-5a00-429f-b90c-611734b324bc", + "name": "service_account", + "description": "Specific scope for a client enabled for service accounts", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "35d9b373-4794-4306-b264-336dfc37e726", + "name": "Client ID", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "client_id", + "id.token.claim": "true", + "introspection.token.claim": "true", + "access.token.claim": "true", + "claim.name": "client_id", + "jsonType.label": "String" + } + }, + { + "id": "4be0bf73-e37f-447d-9c65-caec81826c9b", + "name": "Client Host", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientHost", + "id.token.claim": "true", + "introspection.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientHost", + "jsonType.label": "String" + } + }, + { + "id": "179dc118-75cf-408d-9369-49bbde8557d3", + "name": "Client IP Address", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "clientAddress", + "id.token.claim": "true", + "introspection.token.claim": "true", + "access.token.claim": "true", + "claim.name": "clientAddress", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "33f9a8b9-c57b-4d08-ad64-f78649d6ea55", + "name": "basic", + "description": "OpenID Connect scope for add all basic claims to the token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "f033292b-ae53-4ebd-a6b7-e51ee324a2a1", + "name": "auth_time", + "protocol": "openid-connect", + "protocolMapper": "oidc-usersessionmodel-note-mapper", + "consentRequired": false, + "config": { + "user.session.note": "AUTH_TIME", + "id.token.claim": "true", + "introspection.token.claim": "true", + "access.token.claim": "true", + "claim.name": "auth_time", + "jsonType.label": "long" + } + }, + { + "id": "12d4bacb-cac0-4cae-b033-22de18adcdd2", + "name": "sub", + "protocol": "openid-connect", + "protocolMapper": "oidc-sub-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "access.token.claim": "true" + } + } + ] + }, + { + "id": "f515ec81-3c1b-4d4d-b7a2-e7e8d47b6447", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "consent.screen.text": "${rolesScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "26d299a8-69e2-4864-9595-17a5b417fc61", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "d2998083-a8db-4f4e-9aaa-9cad68d65b97", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "7a4cb2e5-07a0-4c16-a024-71df7ddd6868", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "8f1eafef-92d6-434e-b9ec-6edec1fddd0a", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "c03095aa-b656-447a-9767-0763c2ccb070", + "name": "acr", + "description": "OpenID Connect scope for add acr (authentication context class reference) to the token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "948b230c-56d0-4000-937c-841cd395d3f9", + "name": "acr loa level", + "protocol": "openid-connect", + "protocolMapper": "oidc-acr-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "id": "cdf35f63-8ec7-41a0-ae12-f05d415818cc", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "consent.screen.text": "${phoneScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "ba4348ff-90b1-4e09-89a8-e5c08b04d3d1", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "e6cceae5-8392-4348-b302-f610ece6056e", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "id": "4318001c-2970-41d3-91b9-e31c08569872", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "consent.screen.text": "${emailScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "406d02a6-866a-4962-8838-e8c58ada1505", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "33baabc1-9bf2-42e4-8b8e-a53c13f0b744", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "id": "5277a84f-d727-4c64-8432-d513127beee1", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "consent.screen.text": "${profileScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "0a609875-2678-4056-93ef-dd5c03e6059d", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "7c510d18-07ee-4b78-8acd-24b777d11b3c", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "0bb6d0ea-195f-49e8-918c-c419a26a661c", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "5f1e644c-1acf-440c-b1a6-b5f65bcebfd9", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "c710bdb2-6cfd-4f60-9c4e-730188fc62f7", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "012d5038-0e13-42ba-9df7-2487c8e2eead", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "21590b19-517d-4b6d-92f6-d4f71238677e", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "long", + "userinfo.token.claim": "true" + } + }, + { + "id": "e4cddca7-1360-42f3-9854-da6cbe00c71e", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "afee328f-c64c-43e6-80d0-be2721c2ed0e", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "780a1e2c-5b63-46f4-a5bf-dc3fd8ce0cbb", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "aeebffff-f776-427e-83ed-064707ffce57", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "b3e840a2-1794-4da1-bf69-31905cbff0d6", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "0607e0e4-4f7f-4214-996d-3599772ce1c7", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "426a609b-4e28-4132-af0d-13297b8cb63a", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "id": "a1ebde82-ce21-438f-a3ad-261d3eeb1c01", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "64653ac7-7ffc-4f7c-a589-03e3b68bbd25", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "aeb5b852-dfec-4e67-9d9e-104abe9b3bf2", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "consent.screen.text": "", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "e2fa8437-a0f1-46fc-af9c-c40fc09cd6a1", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "4fecd0d7-d4ad-457e-90f2-c7202bf01ff5", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "a9536634-a9f6-4ed5-a8e7-8379d3b002ca", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String", + "userinfo.token.claim": "true" + } + }, + { + "id": "2ce1a702-9458-4926-9b8a-f82c07215755", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "userinfo.token.claim": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + } + ] + } + ], + "defaultDefaultClientScopes": [ + "role_list", + "profile", + "email", + "roles", + "web-origins", + "acr", + "basic" + ], + "defaultOptionalClientScopes": [ + "offline_access", + "address", + "phone", + "microprofile-jwt" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "referrerPolicy": "no-referrer", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "8115796f-8f1f-4d6a-88f8-ca2938451260", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "044bd055-714d-478e-aa93-303d2161c427", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-address-mapper", + "saml-user-attribute-mapper", + "saml-role-list-mapper", + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-property-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "be465734-3b0f-4370-a144-73db756e23f8", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-user-attribute-mapper", + "oidc-address-mapper", + "oidc-full-name-mapper", + "oidc-usermodel-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "saml-role-list-mapper", + "saml-user-property-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "42a2f64d-ac9e-4221-9cf6-40ff8c868629", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "7ca08915-6c33-454c-88f2-20e1d6553b26", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "f01f2b6f-3f01-4d01-b2f4-70577c6f599c", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "516d7f21-f21a-4690-831e-36ad313093b2", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "c79df6a0-d4d8-4866-b9e6-8ddb5d1bd38e", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + } + ], + "org.keycloak.storage.UserStorageProvider": [ + { + "id": "6b14f48d-4b83-4d6e-bd8d-dc7541124927", + "name": "Dataverse built-in users authentication", + "providerId": "dv-builtin-users-authenticator", + "subComponents": {}, + "config": { + "datasource": [ + "user-store" + ] + } + } + ], + "org.keycloak.userprofile.UserProfileProvider": [ + { + "id": "cf47a21f-c8fb-42f2-9bff-feca967db183", + "providerId": "declarative-user-profile", + "subComponents": {}, + "config": { + "kc.user.profile.config": [ + "{\"attributes\":[{\"name\":\"username\",\"displayName\":\"${username}\",\"validations\":{\"length\":{\"min\":3,\"max\":255},\"username-prohibited-characters\":{},\"up-username-not-idn-homograph\":{}},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"email\",\"displayName\":\"${email}\",\"validations\":{\"email\":{},\"length\":{\"max\":255}},\"required\":{\"roles\":[\"user\"]},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"firstName\",\"displayName\":\"${firstName}\",\"validations\":{\"length\":{\"max\":255},\"person-name-prohibited-characters\":{}},\"required\":{\"roles\":[\"user\"]},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false},{\"name\":\"lastName\",\"displayName\":\"${lastName}\",\"validations\":{\"length\":{\"max\":255},\"person-name-prohibited-characters\":{}},\"required\":{\"roles\":[\"user\"]},\"permissions\":{\"view\":[\"admin\",\"user\"],\"edit\":[\"admin\",\"user\"]},\"multivalued\":false}],\"groups\":[{\"name\":\"user-metadata\",\"displayHeader\":\"User metadata\",\"displayDescription\":\"Attributes, which refer to user metadata\"}],\"unmanagedAttributePolicy\":\"ENABLED\"}" + ] + } + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "6b4a2281-a9e8-43ab-aee7-190ae91b2842", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "68e2d2b0-4976-480f-ab76-f84a17686b05", + "name": "rsa-enc-generated", + "providerId": "rsa-enc-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "RSA-OAEP" + ] + } + }, + { + "id": "728769a3-99a4-4cca-959d-28181dfee7e8", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "33274da1-1e5b-4190-bda4-5912dfde073b", + "name": "hmac-generated-hs512", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS512" + ] + } + }, + { + "id": "f30af2d2-d042-43b8-bc6d-22f6bab6934c", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "authenticationFlows": [ + { + "id": "94c65ba1-ba50-4be2-94c4-de656145eb67", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false + } + ] + }, + { + "id": "9ea0b8f6-882c-45ad-9110-78adf5a5d233", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "99c5ba83-b585-4601-b740-1a26670bf4e9", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "65b73dec-7dd1-4de8-b542-a023b7104afc", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "9a26b76f-da95-43f1-8da3-16c4a0654f07", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Account verification options", + "userSetupAllowed": false + } + ] + }, + { + "id": "0a77285e-d7d5-4b6c-aa9a-3eadb5e7e3d3", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "cb6c0b3b-2f5f-4493-9d14-6130f8b58dd7", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false + } + ] + }, + { + "id": "0fd3db1b-e93d-4768-82ca-a1498ddc11d0", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "86610e70-f9f5-4c11-8a9e-9de1770565fb", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": true, + "flowAlias": "forms", + "userSetupAllowed": false + } + ] + }, + { + "id": "f6aa23dd-8532-4d92-9780-3ea226481e3b", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "4d2caf65-1703-4ddb-8890-70232e91bcd8", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "autheticatorFlow": true, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "eaa20c41-5334-4fb4-8c45-fb9cc71f7f74", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "b9febfb1-f0aa-4590-b782-272a4aa11575", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "User creation or linking", + "userSetupAllowed": false + } + ] + }, + { + "id": "03bb6ff4-eccb-4f2f-8953-3769f78c3bf3", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "autheticatorFlow": true, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "1022f3c2-0469-41c9-861e-918908f103df", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": true, + "flowAlias": "registration form", + "userSetupAllowed": false + } + ] + }, + { + "id": "00d36c3b-e1dc-41f8-bfd0-5f8c80ea07e8", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + }, + { + "id": "4374c16e-8c65-4168-94c2-df1ab3f3e6ad", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "autheticatorFlow": false, + "userSetupAllowed": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "autheticatorFlow": true, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false + } + ] + }, + { + "id": "04d6ed6a-76c9-41fb-9074-bff8a80c2286", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "autheticatorFlow": false, + "userSetupAllowed": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "e7bad67d-1236-430a-a327-9194f9d1e2b0", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "287b5989-a927-4cf5-8067-74594ce19bc1", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "TERMS_AND_CONDITIONS", + "name": "Terms and Conditions", + "providerId": "TERMS_AND_CONDITIONS", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "webauthn-register", + "name": "Webauthn Register", + "providerId": "webauthn-register", + "enabled": true, + "defaultAction": false, + "priority": 70, + "config": {} + }, + { + "alias": "webauthn-register-passwordless", + "name": "Webauthn Register Passwordless", + "providerId": "webauthn-register-passwordless", + "enabled": true, + "defaultAction": false, + "priority": 80, + "config": {} + }, + { + "alias": "delete_credential", + "name": "Delete Credential", + "providerId": "delete_credential", + "enabled": true, + "defaultAction": false, + "priority": 100, + "config": {} + }, + { + "alias": "idp_link", + "name": "Linking Identity Provider", + "providerId": "idp_link", + "enabled": true, + "defaultAction": false, + "priority": 110, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "firstBrokerLoginFlow": "first broker login", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaAuthRequestedUserHint": "login_hint", + "clientOfflineSessionMaxLifespan": "0", + "oauth2DevicePollingInterval": "5", + "clientSessionIdleTimeout": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5", + "realmReusableOtpCode": "false", + "cibaExpiresIn": "120", + "oauth2DeviceCodeLifespan": "600", + "parRequestUriLifespan": "60", + "clientSessionMaxLifespan": "0", + "frontendUrl": "" + }, + "keycloakVersion": "26.3.2", + "userManagedAccessAllowed": false, + "organizationsEnabled": false, + "verifiableCredentialsEnabled": false, + "adminPermissionsEnabled": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + } +} diff --git a/conf/keycloak/test-realm.json b/conf/keycloak/test-realm.json index 2e5ed1c4d69..42432a17a30 100644 --- a/conf/keycloak/test-realm.json +++ b/conf/keycloak/test-realm.json @@ -2060,4 +2060,4 @@ "clientPolicies" : { "policies" : [ ] } -} \ No newline at end of file +} diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml index 5517175d443..34f888acec4 100644 --- a/conf/solr/schema.xml +++ b/conf/solr/schema.xml @@ -242,6 +242,7 @@ + + + + @@ -58,19 +64,9 @@ https://wiki.shibboleth.net/confluence/display/SHIB2/NativeSPConfiguration diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index c6d325a9651..a37819c90e1 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -118,7 +118,7 @@ Moves a dataset whose id is passed to a Dataverse collection whose alias is pass Link a Dataset ^^^^^^^^^^^^^^ -Creates a link between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). :: +Creates a link between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). Accessible to users with Link Dataset permission on the Dataverse collection. :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/datasets/$linked-dataset-id/link/$linking-dataverse-alias @@ -155,7 +155,7 @@ It returns a list in the following format (new format as of v6.4): Unlink a Dataset ^^^^^^^^^^^^^^^^ -Removes a link between a dataset and a Dataverse collection. Accessible to users with Publish Dataset permissions. :: +Removes a link between a dataset and a Dataverse collection. Accessible to users with Link Dataset permission on the Dataverse collection. :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$linked-dataset-id/deleteLink/$linking-dataverse-alias diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index 0bc5fe7395c..8c627609af2 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -145,6 +145,12 @@ experience. .. _quickstart: https://docs.datalad.org/projects/dataverse/en/latest/settingup.html .. _tutorial: https://docs.datalad.org/projects/dataverse/en/latest/tutorial.html +.. _ood-upload: + +Open OnDemand ++++++++++++++ + +`Open OnDemand `_ is a web frontend to High Performance Computing (HPC) resources. Through a system called `OnDemand Loop `_, developed at IQSS, researchers can create datasets in Dataverse and upload files to them from their Open OnDemand installation. They can also :ref:`download ` files from Dataverse. Embedding Data on Websites -------------------------- @@ -220,6 +226,13 @@ Rclone Rclone v1.70+ supports listing and downloading files from Dataverse datasets. For details, consult the Rclone `DOI remote `_ documentation and the `discussion `_ on the Dataverse mailing list. +.. _ood-download: + +Open OnDemand ++++++++++++++ + +`Open OnDemand `_ is a web frontend to High Performance Computing (HPC) resources. Through a system called `OnDemand Loop `_, developed at IQSS, researchers can click a button to explore a dataset (if the :doc:`external tool ` has been enabled) or discover and download datasets from Dataverse installations from around the world (and other data repositories) in their Open OnDemand computational environment. From that environment they can also :ref:`create datasets ` in Dataverse and upload files. + .. _integrations-discovery: Discoverability diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index 200c3a3e342..97baf3e0c8e 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -65,5 +65,5 @@ Two exporters - Schema.org JSONLD and OpenAire - use an algorithm to determine w The Dataverse software implements two jvm-options that can be used to tune the algorithm: -- :ref:`dataverse.personOrOrg.assumeCommaInPersonName` - boolean, default false. If true, Dataverse will assume any name without a comma must be an organization. This may be most useful for curated Dataverse instances that enforce the "family name, given name" convention. -- :ref:`dataverse.personOrOrg.orgPhraseArray` - a JsonArray of strings. Any name that contains one of the strings is assumed to be an organization. For example, "Project" is a word that is not otherwise associated with being an organization. +- :ref:`dataverse.person-or-org.assume-comma-in-person-name` - boolean, default false. If true, Dataverse will assume any name without a comma must be an organization. This may be most useful for curated Dataverse instances that enforce the "family name, given name" convention. +- :ref:`dataverse.person-or-org.org-phrase-array` - a JsonArray of strings. Any name that contains one of the strings is assumed to be an organization. For example, "Project" is a word that is not otherwise associated with being an organization. diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index 93950d70e5c..0c5b9537d20 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -3,7 +3,7 @@ Apps The introduction of Dataverse Software APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. -The apps below are open source and demonstrate how to use Dataverse Software APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse Software APIs in Python, Javascript, R, and Java. +The apps below are open source and demonstrate how to use Dataverse Software APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse Software APIs in Python, Javascript, R, Java, and Ruby. .. contents:: |toctitle| :local: @@ -165,3 +165,13 @@ OpenScholar The Dataverse Software module from OpenScholar allows a Dataverse installation's widgets to be easily embedded in its web pages: https://github.com/openscholar/openscholar/tree/SCHOLAR-3.x/openscholar/modules/os_features/os_dataverse + +Ruby +---- + +Open OnDemand Loop +~~~~~~~~~~~~~~~~~~ + +Open OnDemand Loop integrates Dataverse with Open OnDemand, as :ref:`described ` in the Admin Guide. + +https://github.com/IQSS/ondemand-loop diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 46b4a9e6f00..5be6c78adce 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,14 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.8 +---- + +- For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. +- For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. +- For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. +- For GET /api/externalTools and /api/externalTools/{id} the responses are now formatted as JSON (previously the toolParameters and allowedApiCalls were a JSON object and array (respectively) that were serialized as JSON strings) and any configured "requirements" are included. + v6.7 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 293fc94638d..fa4b4611559 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1400,6 +1400,46 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/access/dataverseFeaturedItemImage/1" +List Templates of a Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Lists the templates for a given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/{ID}/templates" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/1/templates" + +Create a Template for a Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Creates a template for a given Dataverse collection ``id``. + +To create the template, you must send a JSON file. Your JSON file might look like :download:`dataverse-template.json <../_static/api/dataverse-template.json>` which you would send to the Dataverse installation like this: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$SERVER_URL/api/dataverses/{ID}/templates" --upload-file dataverse-template.json + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/1/templates" --upload-file dataverse-template.json + Datasets -------- @@ -2156,26 +2196,26 @@ For these edits your JSON file need only include those dataset fields which you This endpoint also allows removing fields, as long as they are not required by the dataset. To remove a field, send an empty value (``""``) for individual fields. For multiple fields, send an empty array (``[]``). A sample JSON file for removing fields may be downloaded here: :download:`dataset-edit-metadata-delete-fields-sample.json <../_static/api/dataset-edit-metadata-delete-fields-sample.json>` -If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceInternalVersionNumber`` query parameter. This parameter must include the internal version number corresponding to the dataset version being updated. Note that internal version numbers increase sequentially with each version update. +If another user updates the dataset version metadata before you send the update request, metadata inconsistencies may occur. To prevent this, you can use the optional ``sourceLastUpdateTime`` query parameter. This parameter must include the ``lastUpdateTime`` corresponding to the dataset version being updated. The date must be in the format ``yyyy-MM-dd'T'HH:mm:ss'Z'``. -If this parameter is provided, the update will proceed only if the internal version number remains unchanged. Otherwise, the request will fail with an error. +If this parameter is provided, the update will proceed only if the ``lastUpdateTime`` remains unchanged (meaning no one has updated the dataset metadata since you retrieved it). Otherwise, the request will fail with an error. -Example using ``sourceInternalVersionNumber``: +Example using ``sourceLastUpdateTime``: .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z - export SOURCE_INTERNAL_VERSION_NUMBER=5 + export SOURCE_LAST_UPDATE_TIME=2025-04-25T13:58:28Z - curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceInternalVersionNumber=$SOURCE_INTERNAL_VERSION_NUMBER" --upload-file dataset-update-metadata.json + curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceLastUpdateTime=SOURCE_LAST_UPDATE_TIME" --upload-file dataset-update-metadata.json The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceInternalVersionNumber=5" --upload-file dataset-update-metadata.json + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceLastUpdateTime=2025-04-25T13:58:28Z" --upload-file dataset-update-metadata.json Delete Dataset Metadata @@ -3402,10 +3442,62 @@ Archiving is an optional feature that may be configured for a Dataverse installa curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/:persistentId/$VERSION/archivalStatus?persistentId=$PERSISTENT_IDENTIFIER" +Get Dataset External Tool URL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API call generates a URL for accessing an external tool (see :doc:`/installation/external-tools`) that operates at the dataset level. The URL includes necessary authentication tokens and parameters based on the user's permissions and the tool's configuration. + +Authentication is required for draft or deaccessioned datasets and the user must have ViewUnpublishedDataset permission. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV + export TOOL_ID=42 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/datasets/:persistentId/externalTool/$TOOL_ID/toolUrl?persistentId=$PERSISTENT_IDENTIFIER" \ + -H "Content-Type: application/json" \ + -d '{"preview": false, "locale": "en"}' + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/:persistentId/externalTool/42/toolUrl?persistentId=doi:10.5072/FK2/7U7YBV" \ + -H "Content-Type: application/json" \ + -d '{"preview": false, "locale": "en"}' + +The JSON request body accepts the following optional parameters: + +- ``preview``: boolean flag to indicate if the tool should run in preview mode (default: false) +- ``locale``: string specifying the locale for internationalization + +The response includes: + +- ``toolUrl``: the URL to access the external tool +- ``toolName``: the display name of the external tool +- ``datasetId``: the ID of the dataset +- ``preview``: whether the URL is for preview mode + +Example response: + +.. code-block:: json + + { + "status": "OK", + "data": { + "toolUrl": "https://example.com/tool?datasetId=1234&callback=ahr...", + "toolName": "My External Tool", + "datasetId": 1234, + "preview": false + } + } + Get External Tool Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This API call is intended as a callback that can be used by :doc:`/installation/external-tools` to retrieve signed Urls necessary for their interaction with Dataverse. +This API call is intended as a callback that can be used by :doc:`/installation/external-tools` to retrieve signed URLs necessary for their interaction with Dataverse. It can be called directly as well. The response is a JSON object described in the :doc:`/api/external-tools` section of the API guide. @@ -3549,6 +3641,21 @@ See :ref:`:CustomDatasetSummaryFields` in the Installation Guide for how the lis curl "$SERVER_URL/api/datasets/summaryFieldNames" +.. _get-available-dataset-file-categories: + +Get Available Dataset File Categories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This api returns a list of Categories that may be applied to the files of a given dataset. + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG + + curl "$SERVER_URL/api/datasets/:persistentId/availableFileCategories?persistentId=$PERSISTENT_IDENTIFIER" + + .. _guestbook-at-request-api: Configure When a Dataset Guestbook Appears (If Enabled) @@ -4730,6 +4837,8 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. +An optional parameter, sourceLastUpdateTime=datetime (in format: ``yyyy-MM-dd'T'HH:mm:ss'Z'``), can be used to verify that the file metadata being edited has not been changed since you last retrieved it, thereby avoiding potential lost metadata updates. The value for sourceLastUpdateTime can be taken from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call. + A curl example using an ``ID`` .. code-block:: bash @@ -4750,17 +4859,18 @@ The fully expanded example above (without environment variables) looks like this -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ "https://demo.dataverse.org/api/files/24/metadata" -A curl example using a ``PERSISTENT_ID`` +A curl example using a ``PERSISTENT_ID`` and the sourceLastUpdateTime parameter: .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/AAA000 + export UPDATE_TIME=2025-04-25T13:58:28Z curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID" + "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceLastUpdateTime=$UPDATE_TIME" The fully expanded example above (without environment variables) looks like this: @@ -4768,7 +4878,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000" + "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceLastUpdateTime=2025-04-25T13:58:28Z" Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata. @@ -5223,6 +5333,82 @@ Note the optional "limit" parameter. Without it, the API will attempt to populat By default, the admin API calls are blocked and can only be called from localhost. See more details in :ref:`:BlockedApiEndpoints <:BlockedApiEndpoints>` and :ref:`:BlockedApiPolicy <:BlockedApiPolicy>` settings in :doc:`/installation/config`. +Get File External Tool URL +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API call generates a URL for accessing an external tool (see :doc:`/installation/external-tools`) that operates at the file level. The URL includes necessary authentication tokens and parameters based on the user's permissions and the tool's configuration. + +Authentication is required for draft, restricted, embargoed, or expired (retention period) files; the user must have appropriate permissions. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export FILE_ID=42 + export TOOL_ID=3 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$FILE_ID/externalTool/$TOOL_ID/toolUrl" \ + -H "Content-Type: application/json" \ + -d '{"preview": false, "locale": "en"}' + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/42/externalTool/3/toolUrl" \ + -H "Content-Type: application/json" \ + -d '{"preview": false, "locale": "en"}' + +The JSON request body accepts the following optional parameters: + +- ``preview``: boolean flag to indicate if the tool should run in preview mode (default: false) +- ``locale``: string specifying the locale for internationalization + +The response includes: + +- ``toolUrl``: the URL to access the external tool +- ``toolName``: the display name of the external tool +- ``fileId``: the ID of the file +- ``preview``: whether the URL is for preview mode + +Example response: + +.. code-block:: json + + { + "status": "OK", + "data": { + "toolUrl": "https://example.com/tool?fileId=42&callback=ahr...", + "toolName": "File Viewer Tool", + "fileId": 42, + "preview": false + } + } + +Get File External Tool Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API call is intended as a callback that can be used by :doc:`/installation/external-tools` to retrieve signed URLs necessary for their interaction with Dataverse files. +It can be called directly as well. + +The response is a JSON object described in the :doc:`/api/external-tools` section of the API guide. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export FILE_ID=42 + export FILE_METADATA_ID=123 + export TOOL_ID=3 + + curl -H "X-Dataverse-key: $API_TOKEN" -H "Accept:application/json" "$SERVER_URL/api/files/$FILE_ID/metadata/$FILE_METADATA_ID/toolparams/$TOOL_ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Accept:application/json" "https://demo.dataverse.org/api/files/42/metadata/123/toolparams/3" + Get External Tool Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -5775,6 +5961,34 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/info/exportFormats" +Get Customization File Contents +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Customization API is used to retrieve the analytics-code.html as well as other customization file contents. + +See also :ref:`web-analytics-code` in the Configuration section of the Installation Guide and :ref:`Branding Your Installation` + +The Content-Type returned in the header is based on the media type of the file being returned (example: analytics-code.html returns "text/html; charset=UTF-8" + +Valid types are "homePage", "header", "footer", "style", "analytics", and "logo". + +A curl example getting the analytics-code + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export TYPE=analytics + + curl -X GET "$SERVER_URL/api/info/settings/customization/$TYPE" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X GET "https://demo.dataverse.org/api/info/settings/customization/analytics" + +.. _customization-analytics: + .. _metadata-blocks-api: Metadata Blocks @@ -5886,6 +6100,8 @@ Notifications See :ref:`account-notifications` in the User Guide for an overview. For a list of all the notification types mentioned below (e.g. ASSIGNROLE), see :ref:`mute-notifications` in the Admin Guide. +.. _get-all-notifications: + Get All Notifications by User ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -5895,6 +6111,81 @@ Each user can get a dump of their notifications by passing in their API token: curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all" +The expected OK (200) response looks something like this: + +.. code-block:: text + + { + "status": "OK", + "data": { + "notifications": [ + { + "id": 38, + "type": "CREATEACC", + "displayAsRead": true, + "subjectText": "Root: Your account has been created", + "messageText": "Hello, \nWelcome to...", + "sentTimestamp": "2025-07-21T19:15:37Z" + } + ... + +This endpoint supports an optional query parameter ``inAppNotificationFormat`` which, if sent as ``true``, retrieves the fields needed to build the in-app notifications for the Notifications section of the Dataverse UI, omitting fields related to email notifications. + +.. code-block:: bash + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all?inAppNotificationFormat=true" + +The expected OK (200) response looks something like this: + +.. code-block:: text + + { + "status": "OK", + "data": { + "notifications": [ + { + "id": 79, + "type": "CREATEACC", + "displayAsRead": false, + "sentTimestamp": "2025-08-08T08:00:16Z", + "installationBrandName": "Your Installation Name", + "userGuidesBaseUrl": "https://guides.dataverse.org", + "userGuidesVersion": "6.7.1", + "userGuidesSectionPath": "user/index.html" + } + ] + } + } + ... + +Get Unread Count +~~~~~~~~~~~~~~~~ + +You can get a count of your unread notifications as shown below. + +.. code-block:: bash + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/notifications/unreadCount" + +Mark Notification As Read +~~~~~~~~~~~~~~~~~~~~~~~~~ + +After finding the ID of a notification using :ref:`get-all-notifications`, you can pass it to the "markAsRead" API endpoint as shown below. Note that this endpoint is idempotent; you can mark an already-read notification as read over and over. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export NOTIFICATION_ID=555 + + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/notifications/$NOTIFICATION_ID/markAsRead" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/notifications/555/markAsRead" + Delete Notification by User ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -7706,3 +7997,19 @@ Parameters: ``per_page`` Number of results returned per page. +MyData Collection List +~~~~~~~~~~~~~~~~~~~~~~ + +The MyData Collection List API is used to get a list of the collections an authenticated user can create a Dataset in. +Param userIdentifier={userName} is used by a superuser to get the collections for a specific user. + +A curl example listing collections: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList" + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList?userIdentifier=anotherUser" + diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index fd66ec06133..05daf54d43f 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -29,7 +29,7 @@ type string Can be either "dataverse", "dataset", or "file". Multi subtree string The identifier of the Dataverse collection to which the search should be narrowed. The subtree of this Dataverse collection and all its children will be searched. Multiple "subtree" parameters can be used to include multiple Dataverse collections. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats . sort string The sort field. Supported values include "name" and "date". See example under "order". order string The order in which to sort. Can either be "asc" or "desc". For example, https://demo.dataverse.org/api/search?q=data&sort=name&order=asc -per_page int The number of results to return per request. The default is 10. The max is 1000. See :ref:`iteration example `. +per_page int The number of results to return per request. The default is 10. The max is 1000. See :ref:`iteration example `. start int A cursor for paging through search results. See :ref:`iteration example `. show_relevance boolean Whether or not to show details of which fields were matched by the query. False by default. See :ref:`advanced search example `. show_facets boolean Whether or not to show facets that can be operated on by the "fq" parameter. False by default. See :ref:`advanced search example `. @@ -41,6 +41,7 @@ metadata_fields string Includes the requested fields for each dataset in the geo_point string Latitude and longitude in the form ``geo_point=42.3,-71.1``. You must supply ``geo_radius`` as well. See also :ref:`geospatial-search`. geo_radius string Radial distance in kilometers from ``geo_point`` (which must be supplied as well) such as ``geo_radius=1.5``. show_type_counts boolean Whether or not to include total_count_per_object_type for types: Dataverse, Dataset, and Files. +show_collections boolean Whether or not to include a list of parent and linked collections for each dataset search result. search_service string The name of the search service to use for this query. If omitted, the default search service will be used. For available search services, see :ref:`discovering-available-search-services`. ================ ======= =========== @@ -794,7 +795,7 @@ Note: Configurable Search Services are an optional, experimental feature than ma Example API endpoint: https://demo.dataverse.org/api/search/services -This endpoint returns a list of available search services, including their names, and display names. It also indicates the default search service. +This endpoint returns a list of available search services, including their names, and display names. It also indicates the default search service. Example response: diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index e67ed559e0c..abf6ba7379b 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -70,7 +70,7 @@ # built documents. # # The short X.Y version. -version = '6.7.1' +version = '6.8' # The full version, including alpha/beta/rc tags. release = version diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index a4d1da6baea..4adc6bb6fb1 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -109,7 +109,7 @@ It inherits (is built on) an Ubuntu environment from the upstream `base image of Eclipse Temurin `_. You are free to change the JRE/JDK image to your liking (see below). - +.. _base-image-build-instructions: Build Instructions ++++++++++++++++++ diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index 89ae9ac4c2e..46cf95dae54 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -110,14 +110,16 @@ Migrating Datafiles from Local Storage to S3 A number of pilot Dataverse installations start on local storage, then administrators are tasked with migrating datafiles into S3 or similar object stores. The files may be copied with a command-line utility such as `s3cmd `_. You will want to retain the local file hierarchy, keeping the authority (for example: 10.5072) at the bucket "root." -The below example queries may assist with updating dataset and datafile locations in the Dataverse installation's PostgresQL database. Depending on the initial version of the Dataverse Software and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. +The below example queries may assist with updating dataset and datafile locations in the Dataverse installation's PostgresQL database. Depending on the initial version of the Dataverse Software and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. + +In the following queries, ``://`` refers to the ``id`` you have set for the datastore in the configuration (see the :ref:`file-storage` section of the Installation Guide). To Update Dataset Location to S3, Assuming a ``file://`` Prefix ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: - UPDATE dvobject SET storageidentifier=REPLACE(storageidentifier,'file://','s3://') + UPDATE dvobject SET storageidentifier=REPLACE(storageidentifier,'file://','://') WHERE dtype='Dataset'; To Update Datafile Location to your-s3-bucket, Assuming a ``file://`` Prefix @@ -126,17 +128,17 @@ To Update Datafile Location to your-s3-bucket, Assuming a ``file://`` Prefix :: UPDATE dvobject - SET storageidentifier=REPLACE(storageidentifier,'file://','s3://your-s3-bucket:') + SET storageidentifier=REPLACE(storageidentifier,'file://','://your-s3-bucket:') WHERE id IN (SELECT o.id FROM dvobject o, dataset s WHERE o.dtype = 'DataFile' AND s.id = o.owner_id AND s.harvestingclient_id IS null - AND o.storageidentifier NOT LIKE 's3://%'); + AND o.storageidentifier NOT LIKE '://%'); To Update Datafile Location to your-s3-bucket, Assuming no ``file://`` Prefix ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: - UPDATE dvobject SET storageidentifier=CONCAT('s3://your-s3-bucket:', storageidentifier) + UPDATE dvobject SET storageidentifier=CONCAT('://your-s3-bucket:', storageidentifier) WHERE id IN (SELECT o.id FROM dvobject o, dataset s WHERE o.dtype = 'DataFile' AND s.id = o.owner_id AND s.harvestingclient_id IS null AND o.storageidentifier NOT LIKE '%://%'); diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 01bd729a62f..028b80e2892 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -37,7 +37,7 @@ Some of the steps in this document are well-served by having their own dedicated There are a variety of reasons why a step might deserve its own dedicated issue: - The step can be done by a team member other than the person doing the release. -- Stakeholders might be interested in the status of a step (e.g. has the released been deployed to the demo site). +- Stakeholders might be interested in the status of a step (e.g. has the release been deployed to the demo site). Steps don't get their own dedicated issue if it would be confusing to have multiple people involved. Too many cooks in the kitchen, as they say. Also, some steps are so small the overhead of an issue isn't worth it. @@ -54,6 +54,8 @@ Declare a Code Freeze The following steps are made more difficult if code is changing in the "develop" branch. Declare a code freeze until the release is out. Do not allow pull requests to be merged. +For a hotfix, a code freeze (no merging) is necessary not because we want code to stop changing in the branch being hotfix released, but because bumping the version used in Jenkins/Ansible means that API tests will fail in pull requests until the version is bumped in those pull requests. + Conduct Performance Testing --------------------------- @@ -81,27 +83,20 @@ Developers express the need for an addition to release notes by creating a "rele The task at or near release time is to collect these snippets into a single file. - Find the issue in GitHub that tracks the work of creating release notes for the upcoming release. -- Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above. Snippets may not include any issue number or pull request number in the text so be sure copy the number from the filename of the snippet into the final release note. +- Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above. Snippets may not include any issue number or pull request number in the text so be sure to copy the number from the filename of the snippet into the final release note. - Delete (``git rm``) the release note snippets as the content is added to the main release notes file. - Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. -- Take the release notes .md through the regular Code Review and QA process. That is, make a pull request. Here's an example: https://github.com/IQSS/dataverse/pull/10866 +- Make a pull request. Here's an example: https://github.com/IQSS/dataverse/pull/11613 +- Note that we won't merge the release notes until after we have confirmed that the upgrade instructions are valid by performing a couple upgrades. + +For a hotfix, don't worry about release notes yet. Deploy Release Candidate to Internal ------------------------------------ |dedicated| -To upgrade internal, go to /doc/release-notes, open the release-notes.md file for the current release and perform all the steps under "Upgrade Instructions". - -Deploy Release Candidate to Demo --------------------------------- - -|dedicated| - -First, build the release candidate. - -ssh into the dataverse-internal server and undeploy the current war file. -Go to /doc/release-notes, open the release-notes.md file for the current release, and perform all the steps under "Upgrade Instructions". +First, build the release candidate. For a regular release, you will use the "develop" branch, as shown below. For a hotfix, you will use whatever branch name is used for the hotfix. Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the following adjustments to the config: @@ -109,13 +104,27 @@ Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the fo - Branch Specifier (blank for 'any'): ``*/develop`` - Execute shell: Update version in filenames to ``dataverse-5.10.war`` (for example) -Click "Save" then "Build Now". +Click "Save" then "Build Now". The release candidate war file will be available at https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ws/target/ -This will build the war file, and then automatically deploy it on dataverse-internal. Verify that the application has deployed successfully. +ssh into the dataverse-internal server and download the release candidate war file from the URL above. + +Go to /doc/release-notes, open the release-notes.md file for the release we're working on, and perform all the steps under "Upgrade Instructions". Note that for regular releases, we haven't bumped the version yet so you won't be able to follow the steps exactly. (For hotfix releases, the version will be bumped already.) + +Deploy Release Candidate to Demo +-------------------------------- + +|dedicated| + +Deploy the same war file to https://demo.dataverse.org using the same upgrade instructions as above. + +Merge Release Notes (Once Ready) +-------------------------------- + +If the upgrade instructions are perfect, simply merge the release notes. -You can scp the war file to the demo server or download it from https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ws/target/ +If the upgrade instructions aren't quite right, work with the authors of the release notes until they are good enough, and then merge. -ssh into the demo server and follow the upgrade instructions in the release notes. +For a hotfix, there are no release notes to merge yet. Prepare Release Branch ---------------------- @@ -145,18 +154,20 @@ Return to the parent pom and make the following change, which is necessary for p - modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${revision}`` +When testing the version change in Docker note that you will have to build the base image manually. See :ref:`base-image-build-instructions`. + (Before you make this change the value should be ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}``. Later on, after cutting a release, we'll change it back to that value.) -For a regular release, make the changes above in the release branch you created, but hold off for a moment on making a pull requests because Jenkins will fail because it will be testing the previous release. +For a regular release, make the changes above in the release branch you created, but hold off for a moment on making a pull request because Jenkins will fail because it will be testing the previous release. -In the dataverse-ansible repo make bump the version in `jenkins.yml `_ and make a pull request such as https://github.com/gdcc/dataverse-ansible/pull/386. Wait for it to be merged. Note that bumping on the Jenkins side like this will mean that all pull requests will show failures in Jenkins until they are updated to the version we are releasing. +In the dataverse-ansible repo bump the version in `jenkins.yml `_ and make a pull request such as https://github.com/gdcc/dataverse-ansible/pull/386. Wait for it to be merged. Note that bumping on the Jenkins side like this will mean that all pull requests will show failures in Jenkins until they are updated to the version we are releasing. Once dataverse-ansible has been merged, return to the branch you created above ("10852-bump-to-6.4" or whatever) and make a pull request. Ensure that all tests are passing and then put the PR through the normal review and QA process. -If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. +If you are making a hotfix release, ```` should already be set to ``${revision}``. If so, leave it alone. Go ahead and do the normal bumping of version numbers described above. Make the pull request against the "master" branch. Put it through review and QA. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. -Merge "develop" into "master" ------------------------------ +Merge "develop" into "master" (non-hotfix only) +----------------------------------------------- If this is a regular (non-hotfix) release, create a pull request to merge the "develop" branch into the "master" branch using this "compare" link: https://github.com/IQSS/dataverse/compare/master...develop @@ -302,6 +313,8 @@ Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). On the project board at https://github.com/orgs/IQSS/projects/34 edit the tab (view) that shows the milestone to show the next milestone. +.. _base_image_post_release: + Update the Container Base Image Version Property ------------------------------------------------ @@ -311,16 +324,11 @@ Create a new branch (any name is fine but ``prepare-next-iteration`` is suggeste - modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` -Create a pull request and put it through code review, like usual. Give it a milestone of the next release, the one **after** the one we're working on. Once the pull request has been approved, merge it. It should the the first PR merged of the next release. +Create a pull request and put it through code review, like usual. Give it a milestone of the next release, the one **after** the one we're working on. Once the pull request has been approved, merge it. It should be the first PR merged of the next release. For more background, see :ref:`base-image-supported-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 -Lift the Code Freeze and Encourage Developers to Update Their Branches ----------------------------------------------------------------------- - -It's now safe to lift the code freeze. We can start merging pull requests into the "develop" branch for the next release. - -Let developers know that they should merge the latest from the "develop" branch into any branches they are working on. +For a hotfix, we will do this later and in a different branch. See below. Deploy Final Release on Demo ---------------------------- @@ -369,13 +377,33 @@ Announce the Release on Zulip Post a message under #community at https://dataverse.zulipchat.com -For Hotfixes, Merge Hotfix Branch into "develop" and Rename SQL Scripts ------------------------------------------------------------------------ +For Hotfixes, Merge Hotfix Branch into "develop" +------------------------------------------------ + +Note: this only applies to hotfixes! + +We've merged the hotfix into the "master" branch but now we need the fixes (and version bump) in the "develop" branch. + +Make a new branch off the hotfix branch. You can call it something like "6.7.1-merge-hotfix-to-develop". + +In that branch, do the :ref:`base_image_post_release` step you skipped above. Now is the time. + +Create a pull request against develop. Merge conflicts are possible and this pull request should go through review and QA like normal. Afterwards it's fine to delete this branch and the hotfix branch that was merged into master. + +For Hotfixes, Tell Developers to Merge "develop" into Their Branches and Rename SQL Scripts +------------------------------------------------------------------------------------------- Note: this only applies to hotfixes! -We've merged the hotfix into the "master" branch but now we need the fixes (and version bump) in the "develop" branch. Make a new branch off the hotfix branch and create a pull request against develop. Merge conflicts are possible and this pull request should go through review and QA like normal. Afterwards it's fine to delete this branch and the hotfix branch that was merged into master. +Because we have merged a version bump from the hotfix into the "develop" branch, any SQL scripts in the "develop" branch should be renamed (from "5.11.0" to "5.11.1" for example). (To read more about our naming conventions for SQL scripts, see :doc:`sql-upgrade-scripts`.) + +Look at ``src/main/resources/db/migration`` in the "develop" branch and if any SQL scripts have the wrong version, make a pull request (or ask a developer to) to update them (all at once in a single PR is fine). -Because of the hotfix version, any SQL scripts in "develop" should be renamed (from "5.11.0" to "5.11.1" for example). To read more about our naming conventions for SQL scripts, see :doc:`sql-upgrade-scripts`. +Tell developers to merge the "develop" into their open pull requests (to pick up the new version and any fixes) and rename SQL scripts (if any) with the new version. + +Lift the Code Freeze and Encourage Developers to Update Their Branches +---------------------------------------------------------------------- + +It's now safe to lift the code freeze. We can start merging pull requests into the "develop" branch for the next release. -Please note that version bumps and SQL script renaming both require all open pull requests to be updated with the latest from the "develop" branch so you might want to add any SQL script renaming to the hotfix branch before you put it through QA to be merged with develop. This way, open pull requests only need to be updated once. +Let developers know that they should merge the latest from the "develop" branch into any branches they are working on. (For hotfixes we've already told them this.) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 83a6d42c15d..14bf33c9482 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -246,6 +246,8 @@ If you are running an installation with Apache and Payara on the same server, an You should **NOT** use the configuration option above if you are running in a load-balanced environment, or otherwise have the web server on a different host than the application server. +.. _root-collection-permissions: + Root Dataverse Collection Permissions ------------------------------------- @@ -957,7 +959,28 @@ Logging & Slow Performance - When set to true, all JDBC calls will be logged allowing tracing of all JDBC interactions including SQL. - ``false`` +Database Configuration Tips ++++++++++++++++++++++++++++ + +In this section you can find some example scenarios of advanced configuration for the database connection that can improve service performance and availability. +Database Connection Recovery +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Consider the following scenario: if there is no advanced configuration for the database connection and the Dataverse server loses that connection, for example if the database host is down, the server will be "dead" even after the database server is back to normal. +The only solution to recover Dataverse would be to restart the service. To avoid this situation, the following settings can be used to configure validation of the database connection. +This way, the database connection can be automatically recovered after a failure, improving the server availability. For a Docker installation, it is suggested to create an init.d script so that if the container needs to be recreated, these settings will always be configured. + +.. code-block:: bash + + # Enable database connection validation + asadmin create-jvm-options "-Ddataverse.db.is-connection-validation-required=true" + # Configure to use a database table as the validation method + asadmin create-jvm-options "-Ddataverse.db.connection-validation-method=table" + # Configure the "setting" table to be used for connection validation, but any tables can be used + asadmin create-jvm-options "-Ddataverse.db.validation-table-name=setting" + # Configure a validation period of 60 seconds, but different values may be used + asadmin create-jvm-options "-Ddataverse.db.validate-atmost-once-period-in-seconds=60" .. _file-storage: @@ -1420,6 +1443,11 @@ And lastly, to start up the SeaweedFS server and various components you could us weed server -s3 -metricsPort=9327 -dir=/data -s3.config=/config.json +`VAST DataStore `_ + VAST DataStore must be configured with an S3 gateway. A Dataverse bucket must be created. + Follow `VAST DataStore documentation `_ to configure the S3 gateway. + Set ``dataverse.files..path-style-access=true`` since VAST DataStore uses path style access. + **Additional Reported Working S3-Compatible Storage** If you are successfully using an S3 storage implementation not yet listed above, please feel free to @@ -3137,27 +3165,36 @@ This setting is useful in cases such as running your Dataverse installation behi "HTTP_VIA", "REMOTE_ADDR" -.. _dataverse.personOrOrg.assumeCommaInPersonName: +.. _dataverse.person-or-org.assume-comma-in-person-name: -dataverse.personOrOrg.assumeCommaInPersonName -+++++++++++++++++++++++++++++++++++++++++++++ +dataverse.person-or-org.assume-comma-in-person-name ++++++++++++++++++++++++++++++++++++++++++++++++++++ Please note that this setting is experimental. The Schema.org metadata and OpenAIRE exports and the Schema.org metadata included in DatasetPages try to infer whether each entry in the various fields (e.g. Author, Contributor) is a Person or Organization. If you are sure that users are following the guidance to add people in the recommended family name, given name order, with a comma, you can set this true to always assume entries without a comma are for Organizations. The default is false. -.. _dataverse.personOrOrg.orgPhraseArray: +``./asadmin create-jvm-options '-Ddataverse.person-or-org.assume-comma-in-person-name=true'`` -dataverse.personOrOrg.orgPhraseArray -++++++++++++++++++++++++++++++++++++ +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_PERSON_OR_ORG_ASSUME_COMMA_IN_PERSON_NAME``. + +**Note:** This setting was previously called `dataverse.personOrOrg.assumeCommaInPersonName`, which is still available as an alias for backwards compatiblity. + +.. _dataverse.person-or-org.org-phrase-array: + +dataverse.person-or-org.org-phrase-array +++++++++++++++++++++++++++++++++++++++++ Please note that this setting is experimental. The Schema.org metadata and OpenAIRE exports and the Schema.org metadata included in DatasetPages try to infer whether each entry in the various fields (e.g. Author, Contributor) is a Person or Organization. If you have examples where an orgization name is being inferred to belong to a person, you can use this setting to force it to be recognized as an organization. -The value is expected to be a JsonArray of strings. Any name that contains one of the strings is assumed to be an organization. For example, "Project" is a word that is not otherwise associated with being an organization. +The value is expected to be a comma-separated list of strings. Any name that contains one of the strings is assumed to be an organization. For example, "Project" is a word that is not otherwise associated with being an organization. + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_PERSON_OR_ORG_ORG_PHRASE_ARRAY``. +**Note:** This setting was previously called `dataverse.personOrOrg.orgPhraseArray` and expected a JsonArray of strings. Please update both the name and value format if using the old setting. .. _dataverse.api.signature-secret: @@ -3724,7 +3761,13 @@ please find all known feature flags below. Any of these flags can be activated u - Specifies that Terms of Service acceptance is handled by the IdP, eliminating the need to include ToS acceptance boolean parameter (termsAccepted) in the OIDC user registration request body. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. - ``Off`` * - api-bearer-auth-use-builtin-user-on-id-match - - Allows the use of a built-in user account when an identity match is found during API bearer authentication. This feature enables automatic association of an incoming IdP identity with an existing built-in user account, bypassing the need for additional user registration steps. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this feature flag exposes the installation to potential user impersonation issues depending on the specifics of the IdP configured (For example, if it is configured such that an attacker can create a new account in the IdP, or configured social login account, matching a Dataverse built-in account).** + - Allows the use of a built-in user account when an identity match is found during API bearer authentication. This feature enables automatic association of an incoming IdP identity with an existing built-in user account, bypassing the need for additional user registration steps. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this flag could result in impersonation risks if (and only if) used with a misconfigured IdP.** + - ``Off`` + * - api-bearer-auth-use-shib-user-on-id-match + - Allows the use of a Shibboleth user account when an identity match is found during API bearer authentication. This feature enables automatic association of an incoming IdP identity with an existing Shibboleth user account, bypassing the need for additional user registration steps. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this flag could result in impersonation risks if (and only if) used with a misconfigured IdP.** + - ``Off`` + * - api-bearer-auth-use-oauth-user-on-id-match + - Allows the use of an OAuth user account (GitHub, Google, or ORCID) when an identity match is found during API bearer authentication. This feature enables automatic association of an incoming IdP identity with an existing OAuth user account, bypassing the need for additional user registration steps. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this flag could result in impersonation risks if (and only if) used with a misconfigured IdP.** - ``Off`` * - avoid-expensive-solr-join - Changes the way Solr queries are constructed for public content (published Collections, Datasets and Files). It removes a very expensive Solr join on all such documents, improving overall performance, especially for large instances under heavy load. Before this feature flag is enabled, the corresponding indexing feature (see next feature flag) must be turned on and a full reindex performed (otherwise public objects are not going to be shown in search results). See :doc:`/admin/solr-search-index`. @@ -3750,6 +3793,18 @@ please find all known feature flags below. Any of these flags can be activated u * - enable-version-note - Turns on the ability to add/view/edit/delete per-dataset-version notes intended to provide :ref:`provenance` information about why the dataset/version was created. - ``Off`` + * - shibboleth-use-wayfinder + - This flag allows an instance to use Shibboleth with InCommon federation services. Our original Shibboleth implementation that relies on DiscoFeed can no longer be used since InCommon discontinued their old-style metadata feed. An alternative mechanism had to be implemented in order to use WayFinder service, their recommended replacements, instead. + - ``Off`` + * - shibboleth-use-localhost + - A Shibboleth-using Dataverse instance needs to make network calls to the locally-running ``shibd`` service. The default behavior is to use the address configured via the ``siteUrl`` setting. There are however situations (firewalls, etc.) where localhost would be preferable. + - ``Off`` + * - add-local-contexts-permission-check + - Adds a permission check to ensure that the user calling the /api/localcontexts/datasets/{id} API can edit the dataset with that id. This is currently the only use case - see https://github.com/gdcc/dataverse-external-vocab-support/tree/main/packages/local_contexts. The flag adds additional security to stop other uses, but would currently have to be used in conjunction with the api-session-auth feature flag (the security implications of which have not been fully investigated) to still allow adding Local Contexts metadata to a dataset. + - ``Off`` + * - enable-pid-failure-log + - Turns on creation of a monthly log file (logs/PIDFailures_.log) showing failed requests for dataset/file PIDs. Can be used directly or with scripts at https://github.com/gdcc/dataverse-recipes/python/pid_reports to alert admins. + - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development `, you can set them in the `docker compose `_ file. diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index 6a8395c6d12..b8706017e4f 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -159,6 +159,8 @@ Rather than or in addition to specifying individual Identity Providers (see :ref For example, in the United States, you would register your Dataverse installation with `InCommon `_. For a list of federations around the world, see `REFEDS (the Research and Education FEDerations group) `_. The details of how to register with an identity federation are out of scope for this document. +If you are planning to use InCommon, please note that ``shibd`` needs to be configured to use the new MDQ protocol and WayFinder `service `_ `announced `_ `by `_ InCommon. The sample ``shibboleth2.xml`` provided already contains commented-out sections pre-configured to work with this new InCommon framework. Please see https://spaces.at.internet2.edu/display/MDQ/how-to-configure-shib-sp-to-use-mdq and https://spaces.at.internet2.edu/display/federation/how-to-configure-service-to-use-wayfinder for more information. You will also need to set the feature flag ``dataverse.feature.shibboleth-use-wayfinder=true`` (see :ref:`feature-flags`). + For a successful login to Dataverse, certain :ref:`shibboleth-attributes` must be released by the Identity Provider (IdP). Otherwise, in the federation context, users will have the frustrating experience of selecting their IdP in the list but then getting an error like ``Problem with Identity Provider – The SAML assertion for "eppn" was null``. We definitely want to prevent this! There's even some guidance about this problem in the User Guide under the heading :ref:`fix-shib-login` that links back here. For InCommon, a decent strategy for ensuring that IdPs release the necessary attributes is to have both the SP (your Dataverse installation) and the IdP (there are many of these around the world) join the Research & Scholarship (R&S) category. The `R&S website `_ explains the R&S dream well: @@ -259,14 +261,23 @@ On CentOS 6: ``chkconfig shibd on`` -Verify DiscoFeed and Metadata URLs ----------------------------------- +Verify the Metadata URL +----------------------- -As a sanity check, visit the following URLs (substituting your hostname) to make sure you see JSON and XML: +Substitute your hostname and verify that you are seeing your service provider metadata in XML format: -- https://dataverse.example.edu/Shibboleth.sso/DiscoFeed - https://dataverse.example.edu/Shibboleth.sso/Metadata + +If Your Instance is Using Discofeed: Verify DiscoFeed URL +--------------------------------------------------------- + +As another sanity check, substitute your hostname and make sure you see well-formed JSON: + +- https://dataverse.example.edu/Shibboleth.sso/DiscoFeed + +(Skip this step if you'll be using Shibboleth as a registered member of InCommon federation, since the DiscoFeed will not be part of the workflow.) + The JSON in ``DiscoFeed`` comes from the list of IdPs you configured in the ``MetadataProvider`` section of ``shibboleth2.xml`` and will form a dropdown list on the Login Page. Add the Shibboleth Authentication Provider to Your Dataverse Installation diff --git a/doc/sphinx-guides/source/user/appendix.rst b/doc/sphinx-guides/source/user/appendix.rst index a18da372081..d1c46a93fdf 100755 --- a/doc/sphinx-guides/source/user/appendix.rst +++ b/doc/sphinx-guides/source/user/appendix.rst @@ -23,7 +23,9 @@ Supported Metadata Detailed below are what metadata schemas we support for Citation and Domain Specific Metadata in the Dataverse Project: - Citation Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.5 `__, and Dublin Core's `DCMI Metadata Terms `__ . Language field uses `ISO 639-1 `__ controlled vocabulary. -- Geospatial Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.5 `__, and Dublin Core. Country / Nation field uses `ISO 3166-1 `_ controlled vocabulary. +- Geospatial Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.5 `__, and Dublin Core. Country / Nation field uses `ISO 3166-1 `_ controlled vocabulary. + + - Please note that a new and improved Geospatial metadata block is being proposed at ``__. We encourage you to try the block and give feedback in that pull request. - Social Science & Humanities Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, and Dublin Core. - Astronomy and Astrophysics Metadata (`see .tsv `__): These metadata elements can be mapped/exported to the International Virtual Observatory Alliance’s (IVOA) `VOResource Schema format `__ and is based on diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index d88d0a45e68..15376da0896 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -111,7 +111,7 @@ Dataverse installation user accounts can be granted roles that define which acti Roles and permissions may also be granted to groups. Groups can be defined as a collection of Dataverse installation user accounts, a collection of IP addresses (e.g. all users of a library's computers), or a collection of all users who log in using a particular institutional login (e.g. everyone who logs in with a particular university's account credentials). -Admins of a Dataverse collection can assign roles and permissions to the users of that Dataverse collection. If you are an admin on a Dataverse collection, then you will find the link to the Permissions page under the Edit dropdown on the Dataverse collection page. +The user who creates a dataverse is given the "Admin" role on that dataverse (see :ref:`root-collection-permissions`). Admins of a Dataverse collection can assign roles and permissions to the users of that Dataverse collection. If you are an admin on a Dataverse collection, then you will find the link to the Permissions page under the Edit dropdown on the Dataverse collection page. |image2| @@ -215,7 +215,7 @@ Dataset linking allows a Dataverse collection owner to "link" their Dataverse co For example, researchers working on a collaborative study across institutions can each link their own individual institutional Dataverse collections to the one collaborative dataset, making it easier for interested parties from each institution to find the study. -In order to link a dataset, you will need your account to have the "Publish Dataset" permission on the Dataverse collection that is doing the linking. If you created the Dataverse collection then you should have this permission already, but if not then you will need to ask the admin of that Dataverse collection to assign that permission to your account. You do not need any special permissions on the dataset being linked. +In order to link a dataset, you will need your account to have the "Link Dataset" permission on the Dataverse collection that is doing the linking. If you created the Dataverse collection then you should have this permission already, but if not then you will need to ask the admin of that Dataverse collection to assign that permission to your account. You do not need any special permissions on the dataset being linked. To link a dataset to your Dataverse collection, you must navigate to that dataset and click the white "Link" button in the upper-right corner of the dataset page. This will open up a window where you can type in the name of the Dataverse collection that you would like to link the dataset to. Select your Dataverse collection and click the save button. This will establish the link, and the dataset will now appear under your Dataverse collection. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index b0315419959..393b2b07e97 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -8,6 +8,7 @@ This list provides a way to refer to the documentation for previous and future v - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` - |version| +- `6.7.1 `__ - `6.7 `__ - `6.6 `__ - `6.5 `__ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 4a6d455d4de..5936f3dcdab 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -171,7 +171,7 @@ services: dev_keycloak: container_name: "dev_keycloak" - image: 'quay.io/keycloak/keycloak:26.1.4' + image: 'quay.io/keycloak/keycloak:26.3.2' hostname: keycloak environment: - KEYCLOAK_ADMIN=kcadmin diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 6f4e0c066bc..5532cda1a9e 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -20,7 +20,7 @@ ENV SCRIPT_DIR="/scripts" \ ENV PATH="${PATH}:${SCRIPT_DIR}" \ BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap" -ARG PKGS="curl dnsutils dumb-init ed jq netcat-openbsd postgresql-client" +ARG PKGS="bc curl dnsutils dumb-init ed jq netcat-openbsd postgresql-client" # renovate: datasource=github-releases depName=wait4x/wait4x ARG WAIT4X_VERSION="v3.2.0" # renovate: datasource=pypi depName=awscli diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 0a6a95203f1..013034f14d1 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -132,7 +132,7 @@ - 6.7.1 + 6.8 17 UTF-8 @@ -152,15 +152,15 @@ 6.2025.3 42.7.7 9.8.0 - 2.31.3 + 2.33.0 26.30.0 1.7.35 2.19.0 1.2 - 3.12.0 - 1.26.0 + 3.18.0 + 1.28.0 4.5.13 4.4.14 @@ -454,7 +454,7 @@ Once the release has been made (tag created), change this back to "${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}" (These properties are provided by the build-helper plugin below.) --> - + ${revision} ${base.image.version} diff --git a/pom.xml b/pom.xml index c8971fb6587..ceb5ea28d84 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ 1.1.1 1.20.1 5.4.0 - 2.9.2 + 3.2.2 5.5.3 Dataverse API @@ -61,6 +61,10 @@ org.apache.james apache-mime4j-core + + xerces + xercesImpl + @@ -102,6 +106,11 @@ apache-mime4j-dom 0.8.7 + + xerces + xercesImpl + 2.12.2 + @@ -1119,6 +1128,9 @@ Note: During maintenance we will use the app.image.version setting from the DV parent POM. --> unstable + + + false false @@ -1133,6 +1145,9 @@ gdcc/configbaker:${conf.image.tag} ${app.image.tag} + + + noble ubuntu:${conf.image.flavor} false @@ -1191,6 +1206,10 @@ ${app.image.version} + + ${app.image.tag.1} + ${app.image.tag.2} + @ assembly.xml @@ -1226,6 +1245,10 @@ ${app.image.version} + + ${conf.image.tag.1} + ${conf.image.tag.2} + @ ${project.basedir}/modules/container-configbaker/assembly.xml diff --git a/scripts/api/data/metadatablocks/astrophysics.tsv b/scripts/api/data/metadatablocks/astrophysics.tsv index 92792d404c9..1de2d0278f2 100644 --- a/scripts/api/data/metadatablocks/astrophysics.tsv +++ b/scripts/api/data/metadatablocks/astrophysics.tsv @@ -24,7 +24,7 @@ coverage.Polarization Polarization The polarization coverage text 20 FALSE FALSE FALSE FALSE FALSE FALSE astrophysics redshiftType RedshiftType RedshiftType string C "Redshift"; or "Optical" or "Radio" definitions of Doppler velocity used in the data object. text 21 FALSE FALSE FALSE FALSE FALSE FALSE astrophysics resolution.Redshift Redshift Resolution The resolution in redshift (unitless) or Doppler velocity (km/s) in the data object. Enter a floating-point number. float 22 FALSE FALSE FALSE FALSE FALSE FALSE astrophysics - coverage.RedshiftValue Redshift Value The value of the redshift (unitless) or Doppler velocity (km/s in the data object. Enter a floating-point number. float 23 FALSE FALSE TRUE FALSE FALSE FALSE astrophysics + coverage.RedshiftValue Redshift Value The value of the redshift (unitless) or Doppler velocity (km/s in the data object. Enter a floating-point number. none 23 FALSE FALSE TRUE FALSE FALSE FALSE astrophysics coverage.Redshift.MinimumValue Minimum The minimum value of the redshift (unitless) or Doppler velocity (km/s in the data object. Enter a floating-point number. float 24 FALSE FALSE FALSE FALSE FALSE FALSE coverage.RedshiftValue astrophysics coverage.Redshift.MaximumValue Maximum The maximum value of the redshift (unitless) or Doppler velocity (km/s in the data object. Enter a floating-point number. float 25 FALSE FALSE FALSE FALSE FALSE FALSE coverage.RedshiftValue astrophysics #controlledVocabulary DatasetField Value identifier displayOrder @@ -51,4 +51,4 @@ astroType Object 20 astroType Value 21 astroType ValuePair 22 - astroType Survey 23 \ No newline at end of file + astroType Survey 23 diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index dea23aa9a73..b6bed2b9c5b 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -4085,8 +4085,8 @@ language Magoma gmx 3943 gmx language Magori zgr 3944 zgr language Maguindanaon mdh 3945 mdh - language Magɨ (Madang Province) gkd 3946 gkd - language Magɨyi gmg 3947 gmg + language Magi (Madang Province) gkd 3946 gkd + language Magiyi gmg 3947 gmg language Mahali mjx 3948 mjx language Mahasu Pahari bfz 3949 bfz language Mahican mjy 3950 mjy diff --git a/scripts/api/data/role-curator.json b/scripts/api/data/role-curator.json index 91cb7ec43e2..86f18b2ea6a 100644 --- a/scripts/api/data/role-curator.json +++ b/scripts/api/data/role-curator.json @@ -1,13 +1,14 @@ { "alias":"curator", "name":"Curator", - "description":"For datasets, a person who can edit License + Terms, edit Permissions, and publish datasets.", + "description":"For datasets, a person who can edit License + Terms, edit Permissions, and publish and link datasets.", "permissions":[ "ViewUnpublishedDataset", "EditDataset", "DownloadFile", "DeleteDatasetDraft", "PublishDataset", + "LinkDataset", "ManageDatasetPermissions", "ManageFilePermissions", "AddDataverse", diff --git a/src/backports/v6.6/002-pom.xml.patch b/src/backports/v6.6/002-pom.xml.patch index 9d1edeba1c7..a65880a2ed1 100644 --- a/src/backports/v6.6/002-pom.xml.patch +++ b/src/backports/v6.6/002-pom.xml.patch @@ -1,11 +1,21 @@ ---- a/pom.xml (revision 906f874f9fd56241c41e6c2c2f6989c4406f5909) -+++ b/pom.xml (date 1746655334773) -@@ -997,14 +997,18 @@ +--- a/pom.xml ++++ b/pom.xml +@@ -1069,18 +1069,38 @@ + 17 gdcc/dataverse:${app.image.tag} ++ unstable ++ ++ ++ + false false ++ gdcc/base:${base.image.tag} noble @@ -14,13 +24,43 @@ + ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} + + -p${payara.version}-j${target.java.version} ++ gdcc/configbaker:${conf.image.tag} ++ ${app.image.tag} +- ++ ++ ++ ++ alpine ++ alpine:3.18 + false - ++ - -@@ -1046,6 +1050,9 @@ +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1108,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1126,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1139,9 @@ dev_dataverse ${app.image} @@ -30,7 +70,34 @@ ${docker.platforms} -@@ -1075,6 +1082,9 @@ +@@ -1128,26 +1151,33 @@ + + ${base.image} + ++ ++ ${app.image.tag.1} ++ ${app.image.tag.2} ++ + @ + + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + dev_bootstrap ${conf.image} @@ -40,3 +107,24 @@ ${docker.platforms} +@@ -1155,14 +1185,19 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + ++ ++ ${conf.image.tag.1} ++ ${conf.image.tag.2} ++ + @ + + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + + unstable ++ ++ ++ + false + false + +@@ -1133,6 +1136,9 @@ + gdcc/configbaker:${conf.image.tag} + + ${app.image.tag} ++ ++ ++ + noble + ubuntu:${conf.image.flavor} + false +@@ -1191,6 +1197,10 @@ + + ${app.image.version} + ++ ++ ${app.image.tag.1} ++ ${app.image.tag.2} ++ + @ + + assembly.xml +@@ -1226,6 +1236,10 @@ + + ${app.image.version} + ++ ++ ${conf.image.tag.1} ++ ${conf.image.tag.2} ++ + @ + + ${project.basedir}/modules/container-configbaker/assembly.xml diff --git a/src/backports/v6.7/001-pom.xml.patch b/src/backports/v6.7/001-pom.xml.patch new file mode 100644 index 00000000000..1b2001d5546 --- /dev/null +++ b/src/backports/v6.7/001-pom.xml.patch @@ -0,0 +1,44 @@ +--- a/pom.xml ++++ b/pom.xml +@@ -1119,6 +1119,9 @@ + Note: During maintenance we will use the app.image.version setting from the DV parent POM. + --> + unstable ++ ++ ++ + false + false + +@@ -1133,6 +1136,9 @@ + gdcc/configbaker:${conf.image.tag} + + ${app.image.tag} ++ ++ ++ + noble + ubuntu:${conf.image.flavor} + false +@@ -1191,6 +1197,10 @@ + + ${app.image.version} + ++ ++ ${app.image.tag.1} ++ ${app.image.tag.2} ++ + @ + + assembly.xml +@@ -1226,6 +1236,10 @@ + + ${app.image.version} + ++ ++ ${conf.image.tag.1} ++ ${conf.image.tag.2} ++ + @ + + ${project.basedir}/modules/container-configbaker/assembly.xml diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile index beb1de53cd5..e6dbb2f8679 100644 --- a/src/main/docker/Dockerfile +++ b/src/main/docker/Dockerfile @@ -46,8 +46,9 @@ RUN ln -s "${DEPLOY_DIR}/dataverse/supplements/jhove.conf" "${PAYARA_DIR}/glassf ln -s "${DEPLOY_DIR}/dataverse/supplements/jhoveConfig.xsd" "${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}/config/jhoveConfig.xsd" && \ sed -i "${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}/config/jhove.conf" -e "s:/usr/local/payara./glassfish/domains/domain1:${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}:g" -# Workaround for fabric8io/docker-maven-plugin#1865 +# Workaround for fabric8io/docker-maven-plugin#1865 (both arg lines) ARG APP_IMAGE_VERSION +ARG BASE_IMAGE LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.authors="Research Data Management at FZJ " \ org.opencontainers.image.url="https://guides.dataverse.org/en/latest/container/" \ diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java b/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java index 9dd524127d7..658cd3f279c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java @@ -14,14 +14,18 @@ import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.logging.Logger; + import jakarta.servlet.ServletException; import jakarta.servlet.annotation.WebServlet; import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.FileUtil; import jakarta.ejb.EJB; import org.apache.commons.io.IOUtils; +import org.apache.tika.Tika; /** * @@ -29,7 +33,8 @@ */ @WebServlet(name = "CustomizationFilesServlet", urlPatterns = {"/CustomizationFilesServlet"}) public class CustomizationFilesServlet extends HttpServlet { - + private static final Logger logger = Logger.getLogger(CustomizationFilesServlet.class.getCanonicalName()); + @EJB SettingsServiceBean settingsService; @@ -45,7 +50,7 @@ public class CustomizationFilesServlet extends HttpServlet { */ protected void processRequest(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - response.setContentType("text/html;charset=UTF-8"); + response.setContentType(request.getContentType()); String customFileType = request.getParameter("customFileType"); String filePath = getFilePath(customFileType); @@ -56,6 +61,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re try { File fileIn = physicalPath.toFile(); if (fileIn != null) { + String filename = physicalPath.getFileName().toString(); + int dotIndex = filename.lastIndexOf('.'); + String ext = dotIndex >= 0 ? filename.substring(dotIndex) : ""; + String mimeType = FileUtil.lookupFileTypeByExtension(ext); + response.setContentType(mimeType); inputStream = new FileInputStream(fileIn); in = new BufferedReader(new InputStreamReader(inputStream)); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 589fb5fea9c..30d9928f59a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -749,7 +749,7 @@ public JsonObject getCSLJsonFormat() { if (seriesTitles != null) { itemBuilder.containerTitle(formatString(seriesTitles.get(0), true)); } - itemBuilder.version(version).DOI(persistentId.asString()); + itemBuilder.version(version).DOI(persistentId.asRawIdentifier()); if (keywords != null) { itemBuilder .categories(keywords.stream().map(keyword -> formatString(keyword, true)).toArray(String[]::new)); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index a117536cbce..a735ae7470c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -374,6 +374,28 @@ public List getValues_nondisplay() } return returnList; } + /** + * list of values (as opposed to display values). + * used for passing to solr for indexing + */ + public List getDisplayValues() { + List returnList = new ArrayList(); + if (!datasetFieldValues.isEmpty()) { + for (DatasetFieldValue dsfv : datasetFieldValues) { + String value = dsfv.getDisplayValue(); + if (value != null) { + returnList.add(value); + } + } + } else { + for (ControlledVocabularyValue cvv : controlledVocabularyValues) { + if (cvv != null && cvv.getStrValue() != null) { + returnList.add(cvv.getStrValue()); + } + } + } + return returnList; + } /** * appears to be only used for sending info to solr; changed to return values diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index dce7a98fd75..0b6b74e6a73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -63,31 +63,31 @@ public class DatasetFieldServiceBean implements java.io.Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; - + private static final Logger logger = Logger.getLogger(DatasetFieldServiceBean.class.getCanonicalName()); @EJB SettingsServiceBean settingsService; private static final String NAME_QUERY = "SELECT dsfType from DatasetFieldType dsfType where dsfType.name= :fieldName"; - + /* * External vocabulary support: These fields cache information from the CVocConf * setting which controls how Dataverse connects specific metadata block fields * to third-party Javascripts and external vocabulary services to allow users to * input values from a vocabulary(ies) those services manage. */ - - //Configuration json keyed by the id of the 'parent' DatasetFieldType + + //Configuration json keyed by the id of the 'parent' DatasetFieldType Map cvocMap = null; - + //Configuration json keyed by the id of the child DatasetFieldType specified as the 'term-uri-field' //Note that for primitive fields, the prent and term-uri-field are the same and these maps have the same entry Map cvocMapByTermUri = null; - + //Flat list of cvoc term-uri and managed fields by Id Set cvocFieldSet = null; - + //The hash of the existing CVocConf setting. Used to determine when the setting has changed and it needs to be re-parsed to recreate the cvocMaps String oldHash = null; @@ -97,7 +97,7 @@ public List findAllAdvancedSearchFieldTypes() { public List findAllFacetableFieldTypes() { return em.createNamedQuery("DatasetFieldType.findAllFacetable", DatasetFieldType.class) - .getResultList(); + .getResultList(); } public List findFacetableFieldTypesByMetadataBlock(Long metadataBlockId) { @@ -128,7 +128,7 @@ public DatasetFieldType findByName(String name) { } catch (NoResultException e) { return null; } - + } /** @@ -149,11 +149,11 @@ public DatasetFieldType findByNameOpt(String name) { } } - /* + /* * Similar method for looking up foreign metadata field mappings, for metadata - * imports. for these the uniquness of names isn't guaranteed (i.e., there - * can be a field "author" in many different formats that we want to support), - * so these have to be looked up by both the field name and the name of the + * imports. for these the uniquness of names isn't guaranteed (i.e., there + * can be a field "author" in many different formats that we want to support), + * so these have to be looked up by both the field name and the name of the * foreign format. */ public ForeignMetadataFieldMapping findFieldMapping(String formatName, String pathName) { @@ -171,7 +171,7 @@ public ForeignMetadataFieldMapping findFieldMapping(String formatName, String pa public ControlledVocabularyValue findControlledVocabularyValue(Object pk) { return em.find(ControlledVocabularyValue.class, pk); } - + /** * @param dsft The DatasetFieldType in which to look up a * ControlledVocabularyValue. @@ -182,7 +182,7 @@ public ControlledVocabularyValue findControlledVocabularyValue(Object pk) { * @return The ControlledVocabularyValue found or null. */ public ControlledVocabularyValue findControlledVocabularyValueByDatasetFieldTypeAndStrValue(DatasetFieldType dsft, String strValue, boolean lenient) { - TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabularyValue AS o WHERE o.strValue = :strvalue AND o.datasetFieldType = :dsft", ControlledVocabularyValue.class); + TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabularyValue AS o WHERE o.strValue = :strvalue AND o.datasetFieldType = :dsft", ControlledVocabularyValue.class); typedQuery.setParameter("strvalue", strValue); typedQuery.setParameter("dsft", dsft); try { @@ -201,7 +201,7 @@ public ControlledVocabularyValue findControlledVocabularyValueByDatasetFieldType } } } - + public ControlledVocabAlternate findControlledVocabAlternateByControlledVocabularyValueAndStrValue(ControlledVocabularyValue cvv, String strValue){ TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabAlternate AS o WHERE o.strValue = :strvalue AND o.controlledVocabularyValue = :cvv", ControlledVocabAlternate.class); typedQuery.setParameter("strvalue", strValue); @@ -216,7 +216,7 @@ public ControlledVocabAlternate findControlledVocabAlternateByControlledVocabula return (ControlledVocabAlternate) results.get(0); } } - + /** * @param dsft The DatasetFieldType in which to look up a * ControlledVocabularyValue. @@ -226,7 +226,7 @@ public ControlledVocabAlternate findControlledVocabAlternateByControlledVocabula * @return The ControlledVocabularyValue found or null. */ public ControlledVocabularyValue findControlledVocabularyValueByDatasetFieldTypeAndIdentifier (DatasetFieldType dsft, String identifier) { - TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabularyValue AS o WHERE o.identifier = :identifier AND o.datasetFieldType = :dsft", ControlledVocabularyValue.class); + TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM ControlledVocabularyValue AS o WHERE o.identifier = :identifier AND o.datasetFieldType = :dsft", ControlledVocabularyValue.class); typedQuery.setParameter("identifier", identifier); typedQuery.setParameter("dsft", dsft); try { @@ -255,11 +255,11 @@ public MetadataBlock save(MetadataBlock mdb) { public ControlledVocabularyValue save(ControlledVocabularyValue cvv) { return em.merge(cvv); } - + public ControlledVocabAlternate save(ControlledVocabAlternate alt) { return em.merge(alt); - } - + } + /** * This method returns a Map relating DatasetFieldTypes with any external @@ -269,14 +269,14 @@ public ControlledVocabAlternate save(ControlledVocabAlternate alt) { * id or of the child field specified as the 'term-uri-field' (the field where * the URI of the term is stored (and not one of the child fields where the term * name, vocabulary URI, vocabulary Name or other managed information may go.) - * + * * The map only contains values for DatasetFieldTypes that are configured to use external vocabulary services. - * + * * @param byTermUriField - false: the id of the parent DatasetFieldType is the key, true: the 'term-uri-field' DatasetFieldType id is used as the key * @return - a map of JsonObjects containing configuration information keyed by the DatasetFieldType id (Long) */ public Map getCVocConf(boolean byTermUriField){ - + //ToDo - change to an API call to be able to provide feedback if the json is invalid? String cvocSetting = settingsService.getValueForKey(SettingsServiceBean.Key.CVocConf); if (cvocSetting == null || cvocSetting.isEmpty()) { @@ -290,12 +290,12 @@ public Map getCVocConf(boolean byTermUriField){ String newHash = DigestUtils.md5Hex(cvocSetting); if (newHash.equals(oldHash)) { return byTermUriField ? cvocMapByTermUri : cvocMap; - } + } oldHash=newHash; cvocMap=new HashMap<>(); cvocMapByTermUri=new HashMap<>(); cvocFieldSet = new HashSet<>(); - + try (JsonReader jsonReader = Json.createReader(new StringReader(settingsService.getValueForKey(SettingsServiceBean.Key.CVocConf)))) { JsonArray cvocConfJsonArray = jsonReader.readArray(); for (JsonObject jo : cvocConfJsonArray.getValuesAs(JsonObject.class)) { @@ -648,6 +648,7 @@ private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, S logger.fine("RF: " + filtering.toString()); JsonObject managedFields = cvocEntry.getJsonObject("managed-fields"); logger.fine("MF: " + managedFields.toString()); + int nrOfNotFound = 0; for (String filterKey : filtering.keySet()) { if (!filterKey.equals("@context")) { try { @@ -666,9 +667,14 @@ private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, S param = param.substring(1); String[] pathParts = param.split("/"); logger.fine("PP: " + String.join(", ", pathParts)); - JsonValue curPath = readObject; - vals.add(i, processPathSegment(0, pathParts, curPath, termUri)); - logger.fine("Added param value: " + i + ": " + vals.get(i)); + var foundPart = processPathSegment(0, pathParts, readObject, termUri); + if (foundPart == null) { + nrOfNotFound ++ ; + logger.warning("External Vocabulary: no value found for %s - %s".formatted(filterKey, param)); + } else { + vals.add(i, foundPart); + logger.fine("Added param value: " + i + ": " + vals.get(i)); + } } else { logger.fine("Param is: " + param); // param is not a path - either a reference to the term URI @@ -691,19 +697,29 @@ private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, S logger.fine("Added #id pattern: " + filterKey + ": " + termUri); job.add(filterKey, termUri); } else if (pattern.contains("{")) { - if (pattern.equals("{0}")) { - if (vals.get(0) instanceof JsonArray) { - job.add(filterKey, (JsonArray) vals.get(0)); - } else if (vals.get(0) instanceof JsonObject) { - job.add(filterKey, (JsonObject) vals.get(0)); - } else { - job.add(filterKey, (String) vals.get(0)); + if (vals.isEmpty()) { + if (nrOfNotFound == 0) { + logger.warning("External Vocabulary: " + termUri + " - No value found for " + filterKey); + } + } + else { + if (pattern.equals("{0}")) { + if (vals.get(0) instanceof JsonArray) { + job.add(filterKey, (JsonArray) vals.get(0)); + } + else if (vals.get(0) instanceof JsonObject) { + job.add(filterKey, (JsonObject) vals.get(0)); + } + else { + job.add(filterKey, (String) vals.get(0)); + } + } + else { + String result = MessageFormat.format(pattern, vals.toArray()); + logger.fine("Result: " + result); + job.add(filterKey, result); + logger.fine("Added : " + filterKey + ": " + result); } - } else { - String result = MessageFormat.format(pattern, vals.toArray()); - logger.fine("Result: " + result); - job.add(filterKey, result); - logger.fine("Added : " + filterKey + ": " + result); } } else { logger.fine("Added hardcoded pattern: " + filterKey + ": " + pattern); @@ -716,6 +732,9 @@ private JsonObject filterResponse(JsonObject cvocEntry, JsonObject readObject, S } } } + if(nrOfNotFound>0) { + logger.warning("External Vocabulary: " + termUri + " - Failed to find value(s) reported above in " +readObject); + } JsonObject filteredResponse = job.build(); if(filteredResponse.isEmpty()) { logger.severe("Unable to filter response for term: " + termUri + ", received: " + readObject.toString()); @@ -738,13 +757,23 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri if (expected.equals("@id")) { expected = termUri; } - for (int k = 0; k < arr.size(); k++) { - JsonObject jo = arr.getJsonObject(k); - String val = jo.getString(keyVal[0]); - if (val.equals(expected)) { - logger.fine("Found: " + jo.toString()); - curPath = jo; - return processPathSegment(index + 1, pathParts, curPath, termUri); + if (arr != null) { + for (int k = 0; k < arr.size(); k++) { + JsonObject jo = arr.getJsonObject(k); + if (jo!=null) { + JsonValue val = jo.get(keyVal[0]); + if (val != null) { + if (val.getValueType().equals(ValueType.STRING)) { + if (((JsonString) val).getString().equals(expected)) { + logger.fine("Found: " + jo); + curPath = jo; + return processPathSegment(index + 1, pathParts, curPath, termUri); + } + } else { + logger.warning("Expected a string value for " + keyVal[0] + " but found: " + val.getValueType()); + } + } + } } } } else { @@ -765,33 +794,37 @@ Object processPathSegment(int index, String[] pathParts, JsonValue curPath, Stri } else { curPath = ((JsonObject) curPath).get(pathParts[index]); - logger.fine("Found next Path object " + curPath.toString()); + logger.fine("Found next Path object " + curPath); return processPathSegment(index + 1, pathParts, curPath, termUri); } } else { logger.fine("Last segment: " + curPath.toString()); logger.fine("Looking for : " + pathParts[index]); JsonValue jv = ((JsonObject) curPath).get(pathParts[index]); - ValueType type =jv.getValueType(); - if (type.equals(JsonValue.ValueType.STRING)) { - return ((JsonString) jv).getString(); - } else if (jv.getValueType().equals(JsonValue.ValueType.ARRAY)) { - return jv; - } else if (jv.getValueType().equals(JsonValue.ValueType.OBJECT)) { - return jv; + if (jv != null) { + ValueType type = jv.getValueType(); + if (type.equals(ValueType.STRING)) { + return ((JsonString) jv).getString(); + } + else if (jv.getValueType().equals(ValueType.ARRAY)) { + return jv; + } + else if (jv.getValueType().equals(ValueType.OBJECT)) { + return jv; + } } } return null; } - + /** * Supports validation of externally controlled values. If the value is a URI it * must be in the namespace (start with) one of the uriSpace values of an * allowed vocabulary. If free text entries are allowed for this field (per the * configuration), non-uri entries are also assumed valid. - * + * * @param dft * @param value * @return - true: valid @@ -823,7 +856,7 @@ public boolean isValidCVocValue(DatasetFieldType dft, String value) { } return valid; } - + public List getVocabScripts( Map cvocConf) { //ToDo - only return scripts that are needed (those fields are set on display pages, those blocks/fields are allowed in the Dataverse collection for create/edit)? Set scripts = new HashSet(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index a09d42477fc..b41e8d4ac35 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -303,6 +303,7 @@ public enum DisplayMode { private String dropBoxSelection = ""; private String deaccessionReasonText = ""; private String displayCitation; + private String displayTitle; private String deaccessionForwardURLFor = ""; private String showVersionList = "false"; private List