From eb64cde9a802468748e948a4f94e3b5267fc156c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:31:57 -0500 Subject: [PATCH 01/52] initial guestbook apis --- scripts/api/data/guestbook-test.json | 49 ++++++++ .../harvard/iq/dataverse/api/Datasets.java | 82 ++++++++++--- .../harvard/iq/dataverse/api/Guestbooks.java | 102 +++++++++++++++ .../impl/UpdateDatasetGuestbookCommand.java | 50 ++++++++ .../command/impl/UpdateGuestbookCommand.java | 26 ++++ .../iq/dataverse/util/json/JsonParser.java | 89 ++++++++------ .../iq/dataverse/util/json/JsonPrinter.java | 91 ++++++++++---- src/main/java/propertyFiles/Bundle.properties | 3 + .../harvard/iq/dataverse/api/DatasetsIT.java | 116 ++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 112 +++++++++++------ .../dataverse/util/json/JsonParserTest.java | 64 ++++++++++ .../dataverse/util/json/JsonPrinterTest.java | 113 +++++++++++++++-- 12 files changed, 763 insertions(+), 134 deletions(-) create mode 100644 scripts/api/data/guestbook-test.json create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetGuestbookCommand.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateGuestbookCommand.java diff --git a/scripts/api/data/guestbook-test.json b/scripts/api/data/guestbook-test.json new file mode 100644 index 00000000000..710192b510a --- /dev/null +++ b/scripts/api/data/guestbook-test.json @@ -0,0 +1,49 @@ +{ + "name": "my test guestbook", + "enabled": true, + "emailRequired": true, + "nameRequired": true, + "institutionRequired": false, + "positionRequired": false, + "customQuestions": [ + { + "question": "how's your day", + "required": true, + "displayOrder": 0, + "type": "text", + "hidden": false + }, + { + "question": "Describe yourself", + "required": false, + "displayOrder": 1, + "type": "textarea", + "hidden": false + }, + { + "question": "What color car do you drive", + "required": true, + "displayOrder": 2, + "type": "options", + "hidden": false, + "optionValues": [ + { + "value": "Red", + "displayOrder": 0 + }, + { + "value": "White", + "displayOrder": 1 + }, + { + "value": "Yellow", + "displayOrder": 2 + }, + { + "value": "Purple", + "displayOrder": 3 + } + ] + } + ] +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 1b3016ec2f4..f64eec31ef7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -18,17 +18,22 @@ import edu.harvard.iq.dataverse.batch.jobs.importer.ImportMode; import edu.harvard.iq.dataverse.dataaccess.*; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; -import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummary; -import software.amazon.awssdk.services.s3.model.CompletedPart; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; -import edu.harvard.iq.dataverse.dataset.*; +import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; +import edu.harvard.iq.dataverse.dataset.DatasetType; +import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; +import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; import edu.harvard.iq.dataverse.datasetutility.DataFileTagException; import edu.harvard.iq.dataverse.datasetutility.NoFilesException; import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; +import edu.harvard.iq.dataverse.datasetversionsummaries.DatasetVersionSummary; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.exception.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.export.ExportService; import edu.harvard.iq.dataverse.externaltools.ExternalTool; @@ -37,6 +42,7 @@ import edu.harvard.iq.dataverse.globus.GlobusUtil; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.ingest.IngestUtil; +import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.makedatacount.*; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.metrics.MetricsUtil; @@ -67,8 +73,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.*; import jakarta.ws.rs.core.Response.Status; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.util.Strings; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; @@ -78,6 +84,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; +import software.amazon.awssdk.services.s3.model.CompletedPart; import java.io.IOException; import java.io.InputStream; @@ -98,18 +105,11 @@ import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static edu.harvard.iq.dataverse.api.ApiConstants.*; - -import edu.harvard.iq.dataverse.dataset.DatasetType; -import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; -import edu.harvard.iq.dataverse.license.License; +import static edu.harvard.iq.dataverse.api.ApiConstants.*; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; - -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; -import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; -import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; +import static jakarta.ws.rs.core.Response.Status.*; @Path("datasets") public class Datasets extends AbstractApiBean { @@ -128,6 +128,9 @@ public class Datasets extends AbstractApiBean { @EJB GuestbookResponseServiceBean guestbookResponseService; + @EJB + GuestbookServiceBean guestbookService; + @EJB GlobusServiceBean globusService; @@ -5973,6 +5976,57 @@ public Response updateDatasetTypeWithLicenses(@Context ContainerRequestContext c } } + @AuthRequired + @PUT + @Path("{identifier}/guestbook") + public Response updateDatasetGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String body) { + return response(req -> { + Dataset dataset = findDatasetOrDie(identifier); + Long guestbookId = null; + try { + guestbookId = Long.parseLong(body); + final Guestbook guestbook = guestbookService.find(guestbookId); + if (guestbook == null) { + return error(NOT_FOUND, "Could not find a guestbook with id " + guestbookId); + } + + UpdateDatasetGuestbookCommand update_cmd = new UpdateDatasetGuestbookCommand(dataset, guestbook, req); + + commandEngine.submit(update_cmd); + + } catch (NumberFormatException nfe) { + return error(NOT_FOUND, "Could not find a guestbook with id " + guestbookId); + } catch (CommandException ex) { + return error(BAD_REQUEST, ex.getMessage()); + } + return ok("Guestbook " + guestbookId + " set"); + + }, getRequestUser(crc)); + } + + @AuthRequired + @DELETE + @Path("{identifier}/guestbook") + public Response deleteDatasetGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier) { + return response(req -> { + Dataset dataset = findDatasetOrDie(identifier); + if (dataset.getGuestbook() != null) { + Long guestbookId = dataset.getGuestbook().getId(); + try { + UpdateDatasetGuestbookCommand update_cmd = new UpdateDatasetGuestbookCommand(dataset, null, req); + + commandEngine.submit(update_cmd); + + } catch (CommandException ex) { + return error(BAD_REQUEST, ex.getMessage()); + } + return ok("Guestbook removed " + guestbookId); + } else { + return ok("No Guestbook to remove."); + } + }, getRequestUser(crc)); + } + @PUT @AuthRequired @Path("{id}/deleteFiles") diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java new file mode 100644 index 00000000000..ba66ab636e3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java @@ -0,0 +1,102 @@ +package edu.harvard.iq.dataverse.api; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.Guestbook; +import edu.harvard.iq.dataverse.GuestbookServiceBean; +import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookCommand; +import edu.harvard.iq.dataverse.engine.command.impl.UpdateGuestbookCommand; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonUtil; +import jakarta.ejb.EJB; +import jakarta.json.JsonException; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; +import jakarta.ws.rs.*; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.logging.Logger; + +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; + +@Path("guestbooks") +public class Guestbooks extends AbstractApiBean { + + private static final Logger logger = Logger.getLogger(Guestbooks.class.getCanonicalName()); + + @EJB + GuestbookServiceBean guestbookService; + + @GET + @AuthRequired + @Path("{id}") + public Response getGuestbook(@Context ContainerRequestContext crc, @PathParam("id") Long id) { + return response( req -> { + final Guestbook retrieved = guestbookService.find(id); + if (retrieved != null) { + final JsonObjectBuilder jsonbuilder = json(retrieved); + return ok(jsonbuilder); + } else { + return notFound(BundleUtil.getStringFromBundle("dataset.manageGuestbooks.message.notFound")); + } + }, getRequestUser(crc)); + } + + @POST + @AuthRequired + @Path("{identifier}") + public Response createGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String jsonBody) { + return response(req -> { + Dataverse dataverse = findDataverseOrDie(identifier); + logger.severe(">>> jsonBody " + jsonBody); + Guestbook guestbook = new Guestbook(); + guestbook.setDataverse(dataverse); + try { + JsonObject jsonObj = JsonUtil.getJsonObject(jsonBody); + jsonParser().parseGuestbook(jsonObj, guestbook); + } catch (JsonException | JsonParseException ex) { + return badRequest(ex.getMessage()); + } + guestbook.setCreateTime(Timestamp.from(Instant.now())); + execCommand(new CreateGuestbookCommand(guestbook, req, dataverse)); + return ok("Guestbook " + guestbook.getId() + " created"); + }, getRequestUser(crc)); + } + + @PUT + @AuthRequired + @Path("{identifier}/{id}/enabled") + public Response enableGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, @PathParam("id") String id, String body) { + body = body.trim(); + if (!Util.isBoolean(body)) { + return badRequest("Illegal value '" + body + "'. Use 'true' or 'false'"); + } + Long guestbookId; + try { + guestbookId = Long.parseLong(id); + } catch (NumberFormatException nfe) { + return badRequest("Illegal id '" + id + "'"); + } + boolean enabled = Util.isTrue(body); + return response( req -> { + Dataverse dataverse = findDataverseOrDie(identifier); + List guestbooks = dataverse.getGuestbooks(); + if (guestbooks != null) { + for (Guestbook guestbook : guestbooks) { + if (guestbook.getId() == guestbookId) { // Ignore the fact the enable flag might not change. Just return ok + guestbook.setEnabled(enabled); + execCommand(new UpdateGuestbookCommand(guestbook, req, dataverse)); + return ok("Guestbook " + guestbookId + " enabled=" + enabled); + } + } + } + return notFound("Guestbook " + guestbookId + " not found."); + }, getRequestUser(crc)); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetGuestbookCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetGuestbookCommand.java new file mode 100644 index 00000000000..097c5312035 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetGuestbookCommand.java @@ -0,0 +1,50 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Guestbook; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; + +import java.util.List; + +@RequiredPermissions(Permission.EditDataset) +public class UpdateDatasetGuestbookCommand extends AbstractCommand { + + private final Dataset dataset; + private final Guestbook guestbook; + + public UpdateDatasetGuestbookCommand(Dataset dataset, Guestbook guestbook, DataverseRequest aRequest) { + super(aRequest, dataset); + this.dataset = dataset; + this.guestbook = guestbook; + } + + @Override + public Dataset execute(CommandContext ctxt) throws CommandException { + Guestbook allowedGuestbook = null; + // if guestbook is null then we are removing it from the dataset + if (guestbook != null) { + // Make sure the requested guestbook is available via the dataset's ancestry + final List guestbooks = dataset.getOwner().getAvailableGuestbooks(); + for (Guestbook gb : guestbooks) { + if (gb.getId() == guestbook.getId()) { + allowedGuestbook = gb; + break; + } + } + + if (allowedGuestbook == null) { + throw new IllegalCommandException("Could not find an available guestbook with id " + guestbook.getId(), this); + } + } + dataset.setGuestbook(allowedGuestbook); + Dataset savedDataset = ctxt.em().merge(dataset); + ctxt.em().flush(); + return savedDataset; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateGuestbookCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateGuestbookCommand.java new file mode 100644 index 00000000000..2138657ee5b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateGuestbookCommand.java @@ -0,0 +1,26 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.Guestbook; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +@RequiredPermissions( Permission.EditDataverse ) +public class UpdateGuestbookCommand extends AbstractCommand { + + private final Guestbook guestbook; + + public UpdateGuestbookCommand(Guestbook guestbook, DataverseRequest aRequest, Dataverse anAffectedDataverse) { + super(aRequest, anAffectedDataverse); + this.guestbook = guestbook; + } + + @Override + public Guestbook execute(CommandContext ctxt) throws CommandException { + return ctxt.guestbooks().save(guestbook); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 15cb5d7febf..4f1e54ed482 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -1,23 +1,7 @@ package edu.harvard.iq.dataverse.util.json; import com.google.gson.Gson; -import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DataFileCategory; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldConstant; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetFieldValue; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseContact; -import edu.harvard.iq.dataverse.DataverseTheme; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.MetadataBlockServiceBean; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.api.dto.DataverseDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; @@ -37,32 +21,18 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; +import jakarta.json.*; +import jakarta.json.JsonValue.ValueType; import org.apache.commons.validator.routines.DomainValidator; import java.sql.Timestamp; import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; +import java.util.*; import java.util.function.Consumer; +import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; -import jakarta.json.JsonString; -import jakarta.json.JsonValue; -import jakarta.json.JsonValue.ValueType; - /** * Parses JSON objects into domain objects. * @@ -579,6 +549,55 @@ public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) th } } + public Guestbook parseGuestbook(JsonObject obj, Guestbook gb) throws JsonParseException { + try { + gb.setName(obj.getString("name", null)); + gb.setEnabled(obj.getBoolean("enabled")); + gb.setEmailRequired(obj.getBoolean("emailRequired")); + gb.setNameRequired(obj.getBoolean("nameRequired")); + gb.setInstitutionRequired(obj.getBoolean("institutionRequired")); + gb.setPositionRequired(obj.getBoolean("positionRequired")); + gb.setCustomQuestions(parseCustomQuestions(obj.getJsonArray("customQuestions"), gb)); + + gb.setCreateTime(parseDate(obj.getString("createTime", null))); + } catch (ParseException ex) { + throw new JsonParseException(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Arrays.asList(ex.getMessage())) , ex); + } + return gb; + } + private List parseCustomQuestions(JsonArray customQuestions, Guestbook gb) { + final List customQuestionList = (customQuestions != null && !customQuestions.isEmpty()) ? new ArrayList<>() : null; + if (customQuestionList != null) { + customQuestions.forEach(q -> { + JsonObject obj = q.asJsonObject(); + CustomQuestion cq = new CustomQuestion(); + cq.setQuestionString(obj.getString("question")); + cq.setRequired(obj.getBoolean("required")); + cq.setDisplayOrder(obj.getInt("displayOrder")); + cq.setQuestionType(obj.getString("type")); + cq.setHidden(obj.getBoolean("hidden")); + cq.setGuestbook(gb); + + JsonArray optionValues = obj.getJsonArray("optionValues"); + final List cqvList = (optionValues != null && !optionValues.isEmpty()) ? new ArrayList<>() : null; + if (cqvList != null) { + optionValues.forEach(v -> { + JsonObject ov = v.asJsonObject(); + CustomQuestionValue cqv = new CustomQuestionValue(); + cqv.setValueString(ov.getString("value")); + cqv.setDisplayOrder(ov.getInt("displayOrder")); + cqv.setCustomQuestion(cq); + cqvList.add(cqv); + }); + cq.setCustomQuestionValues(cqvList); + } + + customQuestionList.add(cq); + }); + } + return customQuestionList; + } + private edu.harvard.iq.dataverse.license.License parseLicense(String licenseNameOrUri) throws JsonParseException { if (licenseNameOrUri == null){ boolean safeDefaultIfKeyNotFound = true; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 27b7a122c93..687b2dc2122 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1,18 +1,18 @@ package edu.harvard.iq.dataverse.util.json; import edu.harvard.iq.dataverse.*; -import edu.harvard.iq.dataverse.authorization.DataverseRole; -import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup; -import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser; import edu.harvard.iq.dataverse.api.Util; +import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssigneeDisplayInfo; import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; +import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.shib.ShibGroup; import edu.harvard.iq.dataverse.authorization.providers.AuthenticationProviderRow; +import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.branding.BrandingUtil; @@ -21,36 +21,23 @@ import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.datasetversionsummaries.*; -import edu.harvard.iq.dataverse.datavariable.CategoryMetadata; -import edu.harvard.iq.dataverse.datavariable.DataVariable; -import edu.harvard.iq.dataverse.datavariable.SummaryStatistic; -import edu.harvard.iq.dataverse.datavariable.VarGroup; -import edu.harvard.iq.dataverse.datavariable.VariableCategory; -import edu.harvard.iq.dataverse.datavariable.VariableMetadata; -import edu.harvard.iq.dataverse.datavariable.VariableRange; +import edu.harvard.iq.dataverse.datavariable.*; import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; -import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.globus.FileDetailsHolder; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; +import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DatasetFieldWalker; - -import static edu.harvard.iq.dataverse.util.json.FileVersionDifferenceJsonPrinter.jsonFileVersionDifference; -import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; - import edu.harvard.iq.dataverse.util.MailUtil; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; +import jakarta.ejb.EJB; +import jakarta.ejb.Singleton; +import jakarta.json.*; -import java.io.IOException; import java.util.*; - -import jakarta.json.Json; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonObjectBuilder; - import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; @@ -58,12 +45,10 @@ import java.util.logging.Logger; import java.util.stream.Collector; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toList; -import jakarta.ejb.EJB; -import jakarta.ejb.Singleton; -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; +import static edu.harvard.iq.dataverse.util.json.FileVersionDifferenceJsonPrinter.jsonFileVersionDifference; +import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import static java.util.stream.Collectors.toList; /** * Convert objects to Json. @@ -392,6 +377,54 @@ public static JsonObjectBuilder getOwnersFromDvObject(DvObject dvObject){ return getOwnersFromDvObject(dvObject, null); } + public static JsonObjectBuilder json(Guestbook guestbook) { + JsonObjectBuilder guestbookObject = jsonObjectBuilder(); + if (guestbook != null) { + guestbookObject.add("id", guestbook.getId()); + guestbookObject.add("name", guestbook.getName()); + guestbookObject.add("enabled", guestbook.isEnabled()); + guestbookObject.add("emailRequired", guestbook.isEmailRequired()); + guestbookObject.add("nameRequired", guestbook.isNameRequired()); + guestbookObject.add("institutionRequired", guestbook.isInstitutionRequired()); + guestbookObject.add("positionRequired", guestbook.isPositionRequired()); + JsonArrayBuilder customQuestions = Json.createArrayBuilder(); + if (guestbook.getCustomQuestions() != null) { + for (CustomQuestion cq : guestbook.getCustomQuestions()) { + customQuestions.add(json(cq)); + } + } + guestbookObject.add("customQuestions", customQuestions); + if (guestbook.getCreateTime() != null) { + guestbookObject.add("createTime", guestbook.getCreateTime().toString()); + } + if (guestbook.getDataverse() != null) { + guestbookObject.add("dataverseId", guestbook.getDataverse().getId()); + } + } + return guestbookObject; + } + public static JsonObjectBuilder json(CustomQuestion customQuestion) { + JsonObjectBuilder customQuestionObject = jsonObjectBuilder(); + customQuestionObject.add("id", customQuestion.getId()); + customQuestionObject.add("question", customQuestion.getQuestionString()); + customQuestionObject.add("required", customQuestion.isRequired()); + customQuestionObject.add("displayOrder", customQuestion.getDisplayOrder()); + customQuestionObject.add("type", customQuestion.getQuestionType()); + customQuestionObject.add("hidden", customQuestion.isHidden()); + if (customQuestion.getCustomQuestionValues() != null && !customQuestion.getCustomQuestionValues().isEmpty()) { + JsonArrayBuilder customQuestionsValues = Json.createArrayBuilder(); + for (CustomQuestionValue value : customQuestion.getCustomQuestionValues()) { + JsonObjectBuilder customQuestionValueObject = jsonObjectBuilder(); + customQuestionValueObject.add("id", value.getId()); + customQuestionValueObject.add("value", value.getValueString()); + customQuestionValueObject.add("displayOrder", value.getDisplayOrder()); + customQuestionsValues.add(customQuestionValueObject); + } + customQuestionObject.add("optionValues", customQuestionsValues); + } + return customQuestionObject; + } + public static JsonObjectBuilder getOwnersFromDvObject(DvObject dvObject, DatasetVersion dsv) { List ownerList = new ArrayList(); dvObject = dvObject.getOwner(); // We're going to ignore the object itself @@ -477,6 +510,9 @@ public static JsonObjectBuilder json(Dataset ds, Boolean returnOwners) { .add("publisher", BrandingUtil.getInstallationBrandName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) .add("storageIdentifier", ds.getStorageIdentifier()); + if (ds.getGuestbook() != null) { + bld.add("guestbookId", ds.getGuestbook().getId()); + } addDatasetFileCountLimit(ds, bld); if (DvObjectContainer.isMetadataLanguageSet(ds.getMetadataLanguage())) { @@ -547,6 +583,9 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("publicationDate", dataset.getPublicationDateFormattedYYYYMMDD()) .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()) .add("versionNote", dsv.getVersionNote()); + if (dataset.getGuestbook() != null) { + bld.add("guestbookId", dataset.getGuestbook().getId()); + } addDatasetFileCountLimit(dataset, bld); License license = DatasetUtil.getLicense(dsv); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f6c0054a43a..7afce54aac2 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1460,6 +1460,7 @@ dataset.manageGuestbooks.message.enableSuccess=The guestbook has been enabled. dataset.manageGuestbooks.message.enableFailure=The guestbook could not be enabled. dataset.manageGuestbooks.message.disableSuccess=The guestbook has been disabled. dataset.manageGuestbooks.message.disableFailure=The guestbook could not be disabled. +dataset.manageGuestbooks.message.notFound=The guestbook could not be found. dataset.manageGuestbooks.tip.title=Manage Dataset Guestbooks dataset.manageGuestbooks.tip.downloadascsv=Click \"Download All Responses\" to download all collected guestbook responses for this dataverse, as a CSV file. To navigate and analyze your collected responses, we recommend importing this CSV file into Excel, Google Sheets or similar software. dataset.guestbooksResponses.dataset=Dataset @@ -2731,6 +2732,8 @@ guestbook.save.fail=Guestbook Save Failed guestbook.option.msg= - An Option question requires multiple options. Please complete before saving. guestbook.create=The guestbook has been created. guestbook.save=The guestbook has been edited and saved. +#Guestbook API +guestbook.error.parsing=Error parsing Guestbook data. #Shib.java shib.invalidEmailAddress=The SAML assertion contained an invalid email address: "{0}". diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index b7cbb37480c..e297893dac6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -12,7 +12,10 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.json.*; +import edu.harvard.iq.dataverse.util.json.JSONLDUtil; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.xml.XmlUtil; import io.restassured.RestAssured; import io.restassured.http.ContentType; @@ -20,12 +23,7 @@ import io.restassured.path.json.JsonPath; import io.restassured.path.xml.XmlPath; import io.restassured.response.Response; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonValue; -import jakarta.json.JsonArrayBuilder; +import jakarta.json.*; import jakarta.ws.rs.core.Response.Status; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -45,6 +43,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDate; +import java.time.Year; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.logging.Logger; @@ -57,7 +56,6 @@ import static io.restassured.path.json.JsonPath.with; import static jakarta.ws.rs.core.Response.Status.*; import static java.lang.Thread.sleep; -import java.time.Year; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.Matchers.contains; import static org.junit.jupiter.api.Assertions.*; @@ -7413,6 +7411,108 @@ public void testExcludeEmailOverride() { assertTrue(!json.contains("datasetContactEmail")); } + @Test + public void testGetDatasetWithGuestbook() throws IOException { + File guestbookJson = new File("scripts/api/data/guestbook-test.json"); + String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); + String apiToken = getSuperuserToken(); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String ownerAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createGuestbookResponse = UtilIT.createGuestbook(ownerAlias, guestbookAsJson, apiToken); + createGuestbookResponse.prettyPrint(); + JsonPath createdGuestbook = JsonPath.from(createGuestbookResponse.body().asString()); + Long guestbookId = Long.parseLong(createdGuestbook.getString("data.message").split(" ")[1]); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(ownerAlias, apiToken); + createDatasetResponse.prettyPrint(); + String persistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // Enable the Guestbook + Response guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, "x"); + guestbookEnableResponse.prettyPrint(); + guestbookEnableResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", startsWith("Illegal value")); + guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.TRUE.toString()); + guestbookEnableResponse.prettyPrint(); + guestbookEnableResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", startsWith("Guestbook")); + + // Add the Guestbook to the Dataset + Response setGuestbook = UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken); + setGuestbook.prettyPrint(); + + Response getDataset = UtilIT.getDatasetVersions(persistentId, apiToken); + getDataset.prettyPrint(); + getDataset.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data[0].guestbookId", equalTo(guestbookId.intValue())); + + getDataset = UtilIT.nativeGet(datasetId, apiToken); + getDataset.prettyPrint(); + getDataset.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.guestbookId", equalTo(guestbookId.intValue())); + + Response getGuestbook = UtilIT.getGuestbook(Long.valueOf(guestbookId), apiToken); + getGuestbook.prettyPrint(); + getGuestbook.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.id", equalTo(guestbookId.intValue())); + + getGuestbook = UtilIT.getGuestbook(-1L, apiToken); + getGuestbook.prettyPrint(); + getGuestbook.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + + // remove the guestbook from the dataset + Response removeGuestbook = UtilIT.updateDatasetGuestbook(persistentId, null, apiToken); + removeGuestbook.prettyPrint(); + removeGuestbook.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", startsWith("Guestbook removed")); + // remove the already removed guestbook from the dataset + removeGuestbook = UtilIT.updateDatasetGuestbook(persistentId, null, apiToken); + removeGuestbook.prettyPrint(); + removeGuestbook.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", startsWith("No Guestbook to remove")); + + // Get the dataset to show that the guestbook was removed + getDataset = UtilIT.nativeGet(datasetId, apiToken); + getDataset.prettyPrint(); + getDataset.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.guestbookId", equalTo(null)); + + // Disable the Guestbook + guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.FALSE.toString()); + guestbookEnableResponse.prettyPrint(); + guestbookEnableResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", startsWith("Guestbook")); + + // Fail to add a disabled Guestbook to the Dataset + setGuestbook = UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken); + setGuestbook.prettyPrint(); + setGuestbook.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", startsWith("Could not find an available guestbook")); + + // Enable the Guestbook. Add it to the Dataset. Then disable it. + // Show that the guestbook is still returned in the dataset Json even if it's disabled + UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.TRUE.toString()).prettyPrint(); + UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken).prettyPrint(); + UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.FALSE.toString()).prettyPrint(); + getDataset = UtilIT.nativeGet(datasetId, apiToken); + getDataset.prettyPrint(); + getDataset.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.guestbookId", equalTo(guestbookId.intValue())); + } + private String getSuperuserToken() { Response createResponse = UtilIT.createRandomUser(); String adminApiToken = UtilIT.getApiTokenFromResponse(createResponse); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 867bb4f98fc..b7025ac985e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1,56 +1,53 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.*; +import com.mashape.unirest.http.Unirest; +import com.mashape.unirest.http.exceptions.UnirestException; +import com.mashape.unirest.request.GetRequest; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; +import edu.harvard.iq.dataverse.mydata.MyDataFilterParams; +import edu.harvard.iq.dataverse.settings.FeatureFlags; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; -import io.restassured.response.Response; - -import java.io.*; -import java.util.*; -import java.util.logging.Logger; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonObject; - -import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; -import static jakarta.ws.rs.core.Response.Status.CREATED; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.util.logging.Level; -import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; import io.restassured.path.xml.XmlPath; -import edu.harvard.iq.dataverse.mydata.MyDataFilterParams; -import jakarta.ws.rs.core.HttpHeaders; -import org.apache.commons.lang3.StringUtils; -import org.assertj.core.util.Lists; -import org.junit.jupiter.api.Test; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import io.restassured.response.Response; import io.restassured.specification.RequestSpecification; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import com.mashape.unirest.request.GetRequest; -import edu.harvard.iq.dataverse.util.FileUtil; +import jakarta.json.*; +import jakarta.ws.rs.core.HttpHeaders; import org.apache.commons.io.IOUtils; -import java.nio.file.Path; - +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.assertj.core.util.Lists; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; import static edu.harvard.iq.dataverse.api.ApiConstants.*; -import static io.restassured.path.xml.XmlPath.from; +import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import static io.restassured.RestAssured.given; - -import edu.harvard.iq.dataverse.settings.FeatureFlags; -import edu.harvard.iq.dataverse.util.StringUtil; - +import static io.restassured.path.xml.XmlPath.from; +import static jakarta.ws.rs.core.Response.Status.CREATED; import static org.junit.jupiter.api.Assertions.*; public class UtilIT { @@ -595,6 +592,30 @@ static Response getGuestbookResponses(String dataverseAlias, Long guestbookId, S return requestSpec.get("/api/dataverses/" + dataverseAlias + "/guestbookResponses/"); } + public static Response createGuestbook(String dataverseAlias, String guestbookAsJson, String apiToken) { + Response createGuestbookResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(guestbookAsJson) + .contentType("application/json") + .post("/api/guestbooks/" + dataverseAlias); + return createGuestbookResponse; + } + + static Response getGuestbook(Long guestbookId, String apiToken) { + RequestSpecification requestSpec = given() + .header(API_TOKEN_HTTP_HEADER, apiToken); + return requestSpec.get("/api/guestbooks/" + guestbookId ); + } + + static Response enableGuestbook(String dataverseAlias, Long guestbookId, String apiToken, String enable) { + Response createGuestbookResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(enable) + .contentType("application/json") + .put("/api/guestbooks/" + dataverseAlias + "/" + guestbookId + "/enabled"); + return createGuestbookResponse; + } + static Response getCollectionSchema(String dataverseAlias, String apiToken) { Response getCollectionSchemaResponse = given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -808,6 +829,21 @@ static Response updateFieldLevelDatasetMetadataViaNative(String persistentId, St return editVersionMetadataFromJsonStr(persistentId, jsonIn, apiToken, null); } + static Response updateDatasetGuestbook(String persistentId, Long guestbookId, String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json"); + String path = "/api/datasets/:persistentId/guestbook/?persistentId=" + persistentId; + if (guestbookId != null) { + return requestSpecification + .body(guestbookId) + .put(path); + } else { + return requestSpecification + .delete(path); + } + } + static Response editVersionMetadataFromJsonStr(String persistentId, String jsonString, String apiToken) { return editVersionMetadataFromJsonStr(persistentId, jsonString, apiToken, null); } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index d1cb30e2bc3..668389b293b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -733,4 +733,68 @@ public void testEnum() throws JsonParseException { assertTrue(typesSet.contains(Type.REVOKEROLE), "Set contains REVOKEROLE"); assertTrue(typesSet.contains(Type.ASSIGNROLE), "Set contains ASSIGNROLE"); } + + @Test + public void testGuestbook() throws JsonParseException { + final String guestbookJson = """ + { + "name": "my test guestbook", + "enabled": true, + "emailRequired": true, + "nameRequired": true, + "institutionRequired": false, + "positionRequired": false, + "customQuestions": [ + { + "question": "how's your day", + "required": true, + "displayOrder": 0, + "type": "text", + "hidden": false + }, + { + "question": "Describe yourself", + "required": false, + "displayOrder": 1, + "type": "textarea", + "hidden": false + }, + { + "question": "What color car do you drive", + "required": true, + "displayOrder": 2, + "type": "options", + "hidden": false, + "optionValues": [ + { + "value": "Red", + "displayOrder": 0 + }, + { + "value": "White", + "displayOrder": 1 + }, + { + "value": "Yellow", + "displayOrder": 2 + }, + { + "value": "Purple", + "displayOrder": 3 + } + ] + } + ] + } + """; + + JsonObject jsonObj = JsonUtil.getJsonObject(guestbookJson); + Guestbook gb = new Guestbook(); + gb = sut.parseGuestbook(jsonObj, gb); + assertEquals(true, gb.isEnabled()); + assertEquals(3, gb.getCustomQuestions().size()); + assertEquals(4, gb.getCustomQuestions().get(2).getCustomQuestionValues().size()); + assertEquals("Purple", gb.getCustomQuestions().get(2).getCustomQuestionValues().get(3).getValueString()); + assertEquals(3, gb.getCustomQuestions().get(2).getCustomQuestionValues().get(3).getDisplayOrder()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java index 2f4fda068d4..34676335857 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; +import edu.harvard.iq.dataverse.UserNotification.Type; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.RoleAssignee; import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; @@ -12,7 +13,12 @@ import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.UserNotification.Type; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.template.TemplateBuilder; +import jakarta.json.*; +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.sql.Timestamp; import java.time.Instant; @@ -20,19 +26,7 @@ import java.util.*; import java.util.stream.Collectors; -import edu.harvard.iq.dataverse.util.template.TemplateBuilder; - -import jakarta.json.*; - -import edu.harvard.iq.dataverse.util.BundleUtil; -import org.assertj.core.util.Lists; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.BeforeEach; - -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertFalse; public class JsonPrinterTest { @@ -480,6 +474,99 @@ public void testDatasetWithNondefaultType() { assertEquals(sut, result); } + @Test + public void testDatasetWithGuestbook() { + String sut = "foobar"; + DatasetType foobar = new DatasetType(); + foobar.setName(sut); + + Guestbook guestbook = new Guestbook(); + guestbook.setId(1L); + guestbook.setEnabled(true); + guestbook.setName("Test Guestbook"); + guestbook.setEmailRequired(true); + guestbook.setCreateTime(Timestamp.from(Instant.now())); + + int cqOrder = 0; + CustomQuestion cq1 = new CustomQuestion(); + cq1.setDisplayOrder(cqOrder); + cq1.setId(Long.valueOf(++cqOrder)); + cq1.setGuestbook(guestbook); + cq1.setRequired(true); + cq1.setQuestionString("My first question"); + cq1.setQuestionType("text"); // options, textarea, text + + CustomQuestion cq2 = new CustomQuestion(); + cq2.setDisplayOrder(cqOrder); + cq2.setId(Long.valueOf(++cqOrder)); + cq2.setGuestbook(guestbook); + cq2.setRequired(false); + cq2.setQuestionString("My second question"); + cq2.setQuestionType("textarea"); + + CustomQuestion cq3 = new CustomQuestion(); + cq3.setDisplayOrder(cqOrder); + cq3.setId(Long.valueOf(++cqOrder)); + cq3.setGuestbook(guestbook); + cq3.setRequired(false); + cq3.setQuestionString("My third question"); + cq3.setQuestionType("options"); + List values = new ArrayList<>(); + int cqvOrder = 0; + CustomQuestionValue cqv1 = new CustomQuestionValue(); + cqv1.setValueString("Red"); + cqv1.setDisplayOrder(cqvOrder); + cqv1.setId(Long.valueOf(++cqvOrder)); + values.add(cqv1); + CustomQuestionValue cqv2 = new CustomQuestionValue(); + cqv2.setValueString("White"); + cqv2.setDisplayOrder(cqvOrder); + cqv2.setId(Long.valueOf(++cqvOrder)); + values.add(cqv2); + CustomQuestionValue cqv3 = new CustomQuestionValue(); + cqv3.setValueString("Blue"); + cqv3.setDisplayOrder(cqvOrder); + cqv3.setId(Long.valueOf(++cqvOrder)); + values.add(cqv3); + cq3.setCustomQuestionValues(values); + List customQuestions = new ArrayList<>(); + customQuestions.add(cq1); + customQuestions.add(cq2); + customQuestions.add(cq3); + guestbook.setCustomQuestions(customQuestions); + + Dataverse dv = new Dataverse(); + dv.setId(41L); + Dataset dataset = createDataset(42); + dataset.setDatasetType(foobar); + dataset.setOwner(dv); + guestbook.setDataverse(dataset.getOwner()); + dataset.setGuestbook(guestbook); + + // verify that the guestbook id is in the dataset response + var jsob = JsonPrinter.json(dataset.getLatestVersion(), null, false, false, false, false).build(); + System.out.println(jsob); + var gbID = jsob.getInt("guestbookId"); + assertEquals(1, gbID); + + var gb = JsonPrinter.json(guestbook).build(); + System.out.println(gb); + + // verify guestbook values + assertEquals("Test Guestbook", gb.getString("name")); + assertEquals(true, gb.getBoolean("emailRequired")); + assertEquals(false, gb.getBoolean("nameRequired")); + assertEquals(3, gb.getJsonArray("customQuestions").size()); + // verify multiple choice question + var result_cq3 = gb.getJsonArray("customQuestions"); + System.out.println(result_cq3); + var result_cq3_options = result_cq3.getJsonObject(2).getJsonArray("optionValues"); // question 3 is index 2 + System.out.println(result_cq3_options); + assertEquals(3, result_cq3_options.size()); + var result_cq3_options2 = result_cq3_options.getJsonObject(1); // option 2 is index 1 + assertEquals("White", result_cq3_options2.getString("value")); + } + @Test public void testJsonArrayDataverseCollections() { List collections = new ArrayList<>(); From 38d67d483405de98ddfd2d18e98adecda042312c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:54:29 -0500 Subject: [PATCH 02/52] adding guestbook response --- docker-compose-dev.yml | 1 + scripts/api/data/guestbook-test-response.json | 17 +++ .../harvard/iq/dataverse/CustomQuestion.java | 13 +- .../iq/dataverse/CustomQuestionResponse.java | 5 +- .../GuestbookResponseServiceBean.java | 22 +-- .../edu/harvard/iq/dataverse/api/Access.java | 117 +++++++-------- .../harvard/iq/dataverse/api/Guestbooks.java | 1 - .../iq/dataverse/util/json/JsonParser.java | 42 ++++++ src/main/java/propertyFiles/Bundle.properties | 1 + .../harvard/iq/dataverse/api/AccessIT.java | 36 +++-- .../harvard/iq/dataverse/api/DatasetsIT.java | 79 ++++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 83 +++++++++-- .../dataverse/util/json/JsonParserTest.java | 139 +++++++++++------- 13 files changed, 368 insertions(+), 188 deletions(-) create mode 100644 scripts/api/data/guestbook-test-response.json diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 7f12de50b32..95383ea1670 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -54,6 +54,7 @@ services: -Ddataverse.files.minio1.download-redirect=false -Ddataverse.files.minio1.access-key=4cc355_k3y -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y + -Ddataverse.files.guestbook-at-request=true -Ddataverse.pid.providers=fake -Ddataverse.pid.default-provider=fake -Ddataverse.pid.fake.type=FAKE diff --git a/scripts/api/data/guestbook-test-response.json b/scripts/api/data/guestbook-test-response.json new file mode 100644 index 00000000000..df08b52ff6a --- /dev/null +++ b/scripts/api/data/guestbook-test-response.json @@ -0,0 +1,17 @@ +{"guestbookResponse": { + "answers": [ + { + "id": @QID1, + "value": "Good" + }, + { + "id": @QID2, + "value": ["Multi","Line"] + }, + { + "id": @QID3, + "value": "Yellow" + } + ] + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java index d880da5b4a8..a4f36b1bad0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java +++ b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java @@ -1,9 +1,12 @@ package edu.harvard.iq.dataverse; -import java.io.Serializable; -import java.util.List; + import jakarta.persistence.*; import jakarta.validation.constraints.NotBlank; +import java.io.Serializable; +import java.util.List; +import java.util.stream.Collectors; + /** * * @author skraffmiller @@ -92,6 +95,12 @@ public void setQuestionString(String questionString) { public List getCustomQuestionValues() { return customQuestionValues; } + + public List getCustomQuestionOptions() { + return customQuestionValues.stream() + .map(CustomQuestionValue::getValueString) + .collect(Collectors.toList()); + } public String getCustomQuestionValueString(){ String retString = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomQuestionResponse.java b/src/main/java/edu/harvard/iq/dataverse/CustomQuestionResponse.java index f19ee3c3fc7..aead81cd289 100644 --- a/src/main/java/edu/harvard/iq/dataverse/CustomQuestionResponse.java +++ b/src/main/java/edu/harvard/iq/dataverse/CustomQuestionResponse.java @@ -5,11 +5,12 @@ */ package edu.harvard.iq.dataverse; -import java.io.Serializable; -import java.util.List; import jakarta.faces.model.SelectItem; import jakarta.persistence.*; +import java.io.Serializable; +import java.util.List; + /** * * @author skraffmiller diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java index 04ab044cf5e..754fe51714a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java @@ -9,18 +9,6 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.util.StringUtil; -import java.io.IOException; -import java.io.OutputStream; -import java.text.SimpleDateFormat; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.ejb.Stateless; import jakarta.ejb.TransactionAttribute; @@ -30,9 +18,15 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.Query; -import jakarta.persistence.StoredProcedureQuery; import jakarta.persistence.TypedQuery; import org.apache.commons.text.StringEscapeUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.util.*; +import java.util.logging.Logger; /** * * @author skraffmiller @@ -815,7 +809,7 @@ public GuestbookResponse initAPIGuestbookResponse(Dataset dataset, DataFile data } guestbookResponse.setDataset(dataset); guestbookResponse.setResponseTime(new Date()); - guestbookResponse.setSessionId(session.toString()); + guestbookResponse.setSessionId(session != null ? session.toString() : ""); guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); setUserDefaultResponses(guestbookResponse, session, user); return guestbookResponse; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index cadd758a3ac..eb9dbd9abdc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -7,9 +7,6 @@ package edu.harvard.iq.dataverse.api; import edu.harvard.iq.dataverse.*; - -import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; - import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; @@ -17,13 +14,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; -import edu.harvard.iq.dataverse.dataaccess.DataAccess; -import edu.harvard.iq.dataverse.dataaccess.DataAccessRequest; -import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.dataaccess.DataFileZipper; -import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore; -import edu.harvard.iq.dataverse.dataaccess.OptionalAccessService; -import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; +import edu.harvard.iq.dataverse.dataaccess.*; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; @@ -40,64 +31,17 @@ import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; - -import java.util.logging.Logger; import jakarta.ejb.EJB; -import java.io.InputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.logging.Level; import jakarta.inject.Inject; -import jakarta.json.Json; -import java.net.URI; -import jakarta.json.JsonArrayBuilder; +import jakarta.json.*; import jakarta.persistence.TypedQuery; - -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; - -import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.core.Context; -import jakarta.ws.rs.core.HttpHeaders; -import jakarta.ws.rs.core.UriInfo; - - import jakarta.servlet.http.HttpServletResponse; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.ForbiddenException; -import jakarta.ws.rs.NotFoundException; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.QueryParam; -import jakarta.ws.rs.ServiceUnavailableException; -import jakarta.ws.rs.WebApplicationException; -import jakarta.ws.rs.core.Response; -import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; -import jakarta.ws.rs.core.StreamingOutput; -import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; -import java.net.URISyntaxException; - -import jakarta.json.JsonObjectBuilder; -import jakarta.ws.rs.RedirectionException; -import jakarta.ws.rs.ServerErrorException; -import jakarta.ws.rs.core.MediaType; -import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; -import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; - +import jakarta.ws.rs.*; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.*; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; @@ -107,6 +51,21 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataParam; +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; +import static jakarta.ws.rs.core.Response.Status.*; + /* Custom API exceptions [NOT YET IMPLEMENTED] import edu.harvard.iq.dataverse.api.exceptions.NotFoundException; @@ -1394,14 +1353,14 @@ public Response allowAccessRequest(@Context ContainerRequestContext crc, @PathPa * * @param crc * @param fileToRequestAccessId - * @param headers * @return */ @PUT @AuthRequired @Path("/datafile/{id}/requestAccess") - public Response requestFileAccess(@Context ContainerRequestContext crc, @PathParam("id") String fileToRequestAccessId, @Context HttpHeaders headers) { - + public Response requestFileAccess(@Context ContainerRequestContext crc + ,@PathParam("id") String fileToRequestAccessId, String jsonBody) { + DataverseRequest dataverseRequest; DataFile dataFile; @@ -1438,8 +1397,32 @@ public Response requestFileAccess(@Context ContainerRequestContext crc, @PathPar return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.requestExists")); } + // Is Guestbook response required? + // The response will be true (guestbook displays when making a request), false (guestbook displays at download), or will indicate that the dataset inherits one of these settings. + GuestbookResponse guestbookResponse = null; + if (dataFile.getOwner().getEffectiveGuestbookEntryAtRequest()) { + Dataset ds = dataFile.getOwner(); + if (ds.getGuestbook() != null && ds.getGuestbook().isEnabled()) { + // response is required + try { + if (jsonBody == null || jsonBody.isBlank()) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseMissing")); + } + JsonObject jsonObj = JsonUtil.getJsonObject(jsonBody).getJsonObject("guestbookResponse"); + guestbookResponse = guestbookResponseService.initAPIGuestbookResponse(ds, dataFile, null, requestor); + guestbookResponse.setEventType(GuestbookResponse.ACCESS_REQUEST); + // Parse custom question answers + jsonParser().parseGuestbookResponse(jsonObj, guestbookResponse); + engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset())); + } catch (JsonException | JsonParseException | CommandException ex) { + List args = Arrays.asList(dataFile.getDisplayName(), ex.getLocalizedMessage()); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.commandError", args)); + } + } + } + try { - engineSvc.submit(new RequestAccessCommand(dataverseRequest, dataFile, true)); + engineSvc.submit(new RequestAccessCommand(dataverseRequest, dataFile, guestbookResponse, true)); } catch (CommandException ex) { List args = Arrays.asList(dataFile.getDisplayName(), ex.getLocalizedMessage()); return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.commandError", args)); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java index ba66ab636e3..681fa7abd3d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java @@ -54,7 +54,6 @@ public Response getGuestbook(@Context ContainerRequestContext crc, @PathParam("i public Response createGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String jsonBody) { return response(req -> { Dataverse dataverse = findDataverseOrDie(identifier); - logger.severe(">>> jsonBody " + jsonBody); Guestbook guestbook = new Guestbook(); guestbook.setDataverse(dataverse); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 4f1e54ed482..226cec945bf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -598,6 +598,48 @@ private List parseCustomQuestions(JsonArray customQuestions, Gue return customQuestionList; } + public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookResponse guestbookResponse) throws JsonParseException { + + if (obj == null || guestbookResponse == null || guestbookResponse.getGuestbook() == null || guestbookResponse.getGuestbook().getCustomQuestions() == null) { + return null; + } + Map cqMap = new HashMap<>(); + guestbookResponse.getGuestbook().getCustomQuestions().stream().forEach(cq -> cqMap.put(cq.getId(),cq)); + JsonArray answers = obj.getJsonArray("answers"); + List customQuestionResponses = new ArrayList<>(); + for (JsonObject answer : answers.getValuesAs(JsonObject.class)) { + Long cqId = Long.valueOf(answer.getInt("id")); + // find the matching CustomQuestion + CustomQuestion cq = cqMap.get(cqId); + CustomQuestionResponse cqr = new CustomQuestionResponse(); + cqr.setGuestbookResponse(guestbookResponse); + cqr.setCustomQuestion(cq); + String response = null; + if (cq == null) { + throw new JsonParseException("Guestbook Custom Question ID not found!"); + } else if (cq.getQuestionType().equalsIgnoreCase("textarea")) { + String lineFeed = String.valueOf((char) 10); + JsonArray jsonArray = answer.getJsonArray("value"); + List lines = jsonArray.getValuesAs(JsonString.class); + response = lines.stream().map(JsonString::getString).collect(Collectors.joining(lineFeed)); + } else if (cq.getQuestionType().equalsIgnoreCase("options")) { + String option = answer.getString("value"); + if (!cq.getCustomQuestionOptions().contains(option)) { + throw new JsonParseException("Guestbook Custom Question Answer not an option!"); + } + response = option; + } else { + response = answer.getString("value"); + } + cqr.setResponse(response); + customQuestionResponses.add(cqr); + } + guestbookResponse.setCustomQuestionResponses(customQuestionResponses); + // verify each required question is in the response + + return guestbookResponse; + } + private edu.harvard.iq.dataverse.license.License parseLicense(String licenseNameOrUri) throws JsonParseException { if (licenseNameOrUri == null){ boolean safeDefaultIfKeyNotFound = true; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 7afce54aac2..395b6c1e2cf 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2923,6 +2923,7 @@ access.api.requestAccess.failure.commandError=Problem trying request access on { access.api.requestAccess.failure.requestExists=An access request for this file on your behalf already exists. access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. +access.api.requestAccess.failure.guestbookresponseMissing=You may not request access to this file without the required Guestbook response. access.api.requestAccess.noKey=You must provide a key to request access to a file. access.api.requestAccess.fileNotFound=Could not find datafile with id {0}. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index dd8ddd2d315..cee896d4938 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -5,31 +5,31 @@ */ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Guestbook; +import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.json.JsonParseException; import io.restassured.RestAssured; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.util.FileUtil; -import java.io.IOException; -import java.util.zip.ZipInputStream; - -import jakarta.json.Json; +import org.hamcrest.collection.IsMapContaining; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.util.zip.ZipEntry; + import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; - -import org.hamcrest.collection.IsMapContaining; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import static jakarta.ws.rs.core.Response.Status.*; -import static org.hamcrest.MatcherAssert.*; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.*; /** @@ -489,12 +489,17 @@ private HashMap readZipResponse(InputStream iStrea } @Test - public void testRequestAccess() throws InterruptedException { + public void testRequestAccess() throws InterruptedException, IOException, JsonParseException { String pathToJsonFile = "scripts/api/data/dataset-create-new.json"; Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); createDatasetResponse.prettyPrint(); Integer datasetIdNew = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String persistentIdNew = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + + // Create a Guestbook + Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, apiToken); + String guestbookResponseJson = UtilIT.generateGuestbookResponse(guestbook); basicFileName = "004.txt"; String basicPathToFile = "scripts/search/data/replace_test/" + basicFileName; @@ -532,7 +537,16 @@ public void testRequestAccess() throws InterruptedException { Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetIdNew, "major", apiToken); assertEquals(200, publishDataset.getStatusCode()); + // Set the guestbook on the Dataset + UtilIT.updateDatasetGuestbook(persistentIdNew, guestbook.getId(), apiToken).prettyPrint(); + // Request file access WITHOUT the required Guestbook Response (getEffectiveGuestbookEntryAtRequest) + UtilIT.setGuestbookEntryOnRequest(datasetId.toString(), apiToken, Boolean.TRUE).prettyPrint(); requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); + requestFileAccessResponse.prettyPrint(); + assertEquals(400, requestFileAccessResponse.getStatusCode()); + // Request file access with the required Guestbook Response (getEffectiveGuestbookEntryAtRequest) + requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando, guestbookResponseJson); + requestFileAccessResponse.prettyPrint(); assertEquals(200, requestFileAccessResponse.getStatusCode()); Response listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index e297893dac6..97e3494c937 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -7412,56 +7412,43 @@ public void testExcludeEmailOverride() { } @Test - public void testGetDatasetWithGuestbook() throws IOException { - File guestbookJson = new File("scripts/api/data/guestbook-test.json"); - String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); + public void testGetDatasetWithGuestbook() throws IOException, JsonParseException { String apiToken = getSuperuserToken(); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); String ownerAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - Response createGuestbookResponse = UtilIT.createGuestbook(ownerAlias, guestbookAsJson, apiToken); - createGuestbookResponse.prettyPrint(); - JsonPath createdGuestbook = JsonPath.from(createGuestbookResponse.body().asString()); - Long guestbookId = Long.parseLong(createdGuestbook.getString("data.message").split(" ")[1]); - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(ownerAlias, apiToken); createDatasetResponse.prettyPrint(); String persistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); - // Enable the Guestbook - Response guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, "x"); + // Create a Guestbook + Guestbook guestbook = UtilIT.createRandomGuestbook(ownerAlias, persistentId, apiToken); + + // Enable the Guestbook with invalid enable flag + Response guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbook.getId(), apiToken, "x"); guestbookEnableResponse.prettyPrint(); guestbookEnableResponse.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) .body("message", startsWith("Illegal value")); - guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.TRUE.toString()); - guestbookEnableResponse.prettyPrint(); - guestbookEnableResponse.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.message", startsWith("Guestbook")); - - // Add the Guestbook to the Dataset - Response setGuestbook = UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken); - setGuestbook.prettyPrint(); Response getDataset = UtilIT.getDatasetVersions(persistentId, apiToken); getDataset.prettyPrint(); getDataset.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data[0].guestbookId", equalTo(guestbookId.intValue())); + .body("data[0].guestbookId", equalTo(guestbook.getId().intValue())); getDataset = UtilIT.nativeGet(datasetId, apiToken); getDataset.prettyPrint(); getDataset.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.guestbookId", equalTo(guestbookId.intValue())); + .body("data.guestbookId", equalTo(guestbook.getId().intValue())); - Response getGuestbook = UtilIT.getGuestbook(Long.valueOf(guestbookId), apiToken); + Response getGuestbook = UtilIT.getGuestbook(guestbook.getId(), apiToken); getGuestbook.prettyPrint(); getGuestbook.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.id", equalTo(guestbookId.intValue())); + .body("data.id", equalTo(guestbook.getId().intValue())); getGuestbook = UtilIT.getGuestbook(-1L, apiToken); getGuestbook.prettyPrint(); @@ -7488,14 +7475,14 @@ public void testGetDatasetWithGuestbook() throws IOException { .body("data.guestbookId", equalTo(null)); // Disable the Guestbook - guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.FALSE.toString()); + guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbook.getId(), apiToken, Boolean.FALSE.toString()); guestbookEnableResponse.prettyPrint(); guestbookEnableResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.message", startsWith("Guestbook")); // Fail to add a disabled Guestbook to the Dataset - setGuestbook = UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken); + Response setGuestbook = UtilIT.updateDatasetGuestbook(persistentId, guestbook.getId(), apiToken); setGuestbook.prettyPrint(); setGuestbook.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) @@ -7503,14 +7490,48 @@ public void testGetDatasetWithGuestbook() throws IOException { // Enable the Guestbook. Add it to the Dataset. Then disable it. // Show that the guestbook is still returned in the dataset Json even if it's disabled - UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.TRUE.toString()).prettyPrint(); - UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken).prettyPrint(); - UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.FALSE.toString()).prettyPrint(); + UtilIT.enableGuestbook(ownerAlias, guestbook.getId(), apiToken, Boolean.TRUE.toString()).prettyPrint(); + UtilIT.updateDatasetGuestbook(persistentId, guestbook.getId(), apiToken).prettyPrint(); + UtilIT.enableGuestbook(ownerAlias, guestbook.getId(), apiToken, Boolean.FALSE.toString()).prettyPrint(); getDataset = UtilIT.nativeGet(datasetId, apiToken); getDataset.prettyPrint(); getDataset.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.guestbookId", equalTo(guestbookId.intValue())); + .body("data.guestbookId", equalTo(guestbook.getId().intValue())); + } + + @Test + public void testGetDatasetWithTermsOfUseAndGuestbook() throws IOException, JsonParseException { + File guestbookJson = new File("scripts/api/data/guestbook-test.json"); + String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); + // Create users, Dataverse, and Dataset + String adminApiToken = getSuperuserToken(); + Response createResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); + String ownerAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(ownerAlias, adminApiToken); + createDatasetResponse.prettyPrint(); + String persistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + // Create a Guestbook + Guestbook guestbook = UtilIT.createRandomGuestbook(ownerAlias, persistentId, adminApiToken); + // Create a license for Terms of Use + String jsonString = """ + { + "customTerms": { + "termsOfUse": "testTermsOfUse" + } + } + """; + Response updateLicenseResponse = UtilIT.updateLicense(datasetId.toString(), jsonString, adminApiToken); + updateLicenseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo(BundleUtil.getStringFromBundle("datasets.api.updateLicense.success"))); + + // Publish + UtilIT.publishDataverseViaNativeApi(ownerAlias, adminApiToken).prettyPrint(); + UtilIT.publishDatasetViaNativeApi(persistentId, "major", adminApiToken).prettyPrint(); } private String getSuperuserToken() { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index d49888e5be5..38ae66d1f58 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3,16 +3,16 @@ import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import com.mashape.unirest.request.GetRequest; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetFieldValue; -import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; import edu.harvard.iq.dataverse.mydata.MyDataFilterParams; import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonParser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; @@ -48,6 +48,8 @@ import static io.restassured.RestAssured.given; import static io.restassured.path.xml.XmlPath.from; import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.hamcrest.CoreMatchers.startsWith; import static org.junit.jupiter.api.Assertions.*; public class UtilIT { @@ -2111,8 +2113,11 @@ static Response allowAccessRequests(String datasetIdOrPersistentId, boolean allo } static Response requestFileAccess(String fileIdOrPersistentId, String apiToken) { - System.out.print ("Reuest file acceess + fileIdOrPersistentId: " + fileIdOrPersistentId); - System.out.print ("Reuest file acceess + apiToken: " + apiToken); + return requestFileAccess(fileIdOrPersistentId, apiToken, null); + } + static Response requestFileAccess(String fileIdOrPersistentId, String apiToken, String body) { + System.out.print ("Request file access + fileIdOrPersistentId: " + fileIdOrPersistentId); + System.out.print ("Request file access + apiToken: " + apiToken); String idInPath = fileIdOrPersistentId; // Assume it's a number. String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (!NumberUtils.isCreatable(fileIdOrPersistentId)) { @@ -2124,10 +2129,16 @@ static Response requestFileAccess(String fileIdOrPersistentId, String apiToken) if (optionalQueryParam.isEmpty()) { keySeparator = "?"; } - System.out.print ("URL: " + "/api/access/datafile/" + idInPath + "/requestAccess" + optionalQueryParam + keySeparator + "key=" + apiToken); - Response response = given() - .put("/api/access/datafile/" + idInPath + "/requestAccess" + optionalQueryParam + keySeparator + "key=" + apiToken); - return response; + String path = "/api/access/datafile/" + idInPath + "/requestAccess" + optionalQueryParam + keySeparator + "key=" + apiToken; + System.out.print ("URL: " + path); + RequestSpecification requestSpecification = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json"); + if (body != null) { + requestSpecification.body(body); + } + + return requestSpecification.put(path); } static Response grantFileAccess(String fileIdOrPersistentId, String identifier, String apiToken) { @@ -5356,4 +5367,56 @@ public static Response sendMessageToLDNInbox(String message) { .when() .post("/api/inbox/"); } + + public static Response setGuestbookEntryOnRequest(String datasetId, String apiToken, Boolean enabled) { + return given() + .body(enabled) + .contentType(ContentType.JSON) + .header(API_TOKEN_HTTP_HEADER, apiToken) + .put("/api/datasets/" + datasetId + "/guestbookEntryAtRequest"); + } + + public static Guestbook createRandomGuestbook(String ownerAlias, String persistentId, String apiToken) throws IOException, JsonParseException { + Guestbook gb = new Guestbook(); + File guestbookJson = new File("scripts/api/data/guestbook-test.json"); + String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); + JsonObject jsonObj = JsonUtil.getJsonObject(guestbookAsJson); + JsonParser jsonParsor = new JsonParser(); + jsonParsor.parseGuestbook(jsonObj, gb); + + Response createGuestbookResponse = UtilIT.createGuestbook(ownerAlias, guestbookAsJson, apiToken); + createGuestbookResponse.prettyPrint(); + JsonPath createdGuestbook = JsonPath.from(createGuestbookResponse.body().asString()); + Long guestbookId = Long.parseLong(createdGuestbook.getString("data.message").split(" ")[1]); + Response guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbookId, apiToken, Boolean.TRUE.toString()); + guestbookEnableResponse.prettyPrint(); + guestbookEnableResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", startsWith("Guestbook")); + Response getGuestbookResponse = UtilIT.getGuestbook(guestbookId, apiToken); + getGuestbookResponse.prettyPrint(); + JsonPath jsonPath = JsonPath.from(getGuestbookResponse.body().asString()); + gb.setId(guestbookId); + gb.getCustomQuestions().get(0).setId(jsonPath.getLong("data.customQuestions[0].id")); + gb.getCustomQuestions().get(1).setId(jsonPath.getLong("data.customQuestions[1].id")); + gb.getCustomQuestions().get(2).setId(jsonPath.getLong("data.customQuestions[2].id")); + + // Add the Guestbook to the Dataset + Response setGuestbook = UtilIT.updateDatasetGuestbook(persistentId, guestbookId, apiToken); + setGuestbook.prettyPrint(); + return gb; + } + + public static String generateGuestbookResponse(Guestbook gb) throws IOException { + File guestbookJson = new File("scripts/api/data/guestbook-test-response.json"); + String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); + + List cqIDs = new ArrayList<>(); + gb.getCustomQuestions().stream().forEach(cq -> cqIDs.add(cq.getId())); + + return guestbookAsJson.replace("@ID", gb.getId().toString()) + .replace("@QID1", cqIDs.get(0).toString()) + .replace("@QID2", cqIDs.get(1).toString()) + .replace("@QID3", cqIDs.get(2).toString()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index 668389b293b..668a7e45995 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -65,6 +65,58 @@ public class JsonParserTest { DatasetFieldType pubIdType; DatasetFieldType compoundSingleType; JsonParser sut; + + final static String guestbookJson = """ + { + "name": "my test guestbook", + "enabled": true, + "emailRequired": true, + "nameRequired": true, + "institutionRequired": false, + "positionRequired": false, + "customQuestions": [ + { + "question": "how's your day", + "required": true, + "displayOrder": 0, + "type": "text", + "hidden": false + }, + { + "question": "Describe yourself", + "required": false, + "displayOrder": 1, + "type": "textarea", + "hidden": false + }, + { + "question": "What color car do you drive", + "required": true, + "displayOrder": 2, + "type": "options", + "hidden": false, + "optionValues": [ + { + "value": "Red", + "displayOrder": 0 + }, + { + "value": "White", + "displayOrder": 1 + }, + { + "value": "Yellow", + "displayOrder": 2 + }, + { + "value": "Purple", + "displayOrder": 3 + } + ] + } + ] + } + """; public JsonParserTest() { } @@ -736,58 +788,6 @@ public void testEnum() throws JsonParseException { @Test public void testGuestbook() throws JsonParseException { - final String guestbookJson = """ - { - "name": "my test guestbook", - "enabled": true, - "emailRequired": true, - "nameRequired": true, - "institutionRequired": false, - "positionRequired": false, - "customQuestions": [ - { - "question": "how's your day", - "required": true, - "displayOrder": 0, - "type": "text", - "hidden": false - }, - { - "question": "Describe yourself", - "required": false, - "displayOrder": 1, - "type": "textarea", - "hidden": false - }, - { - "question": "What color car do you drive", - "required": true, - "displayOrder": 2, - "type": "options", - "hidden": false, - "optionValues": [ - { - "value": "Red", - "displayOrder": 0 - }, - { - "value": "White", - "displayOrder": 1 - }, - { - "value": "Yellow", - "displayOrder": 2 - }, - { - "value": "Purple", - "displayOrder": 3 - } - ] - } - ] - } - """; - JsonObject jsonObj = JsonUtil.getJsonObject(guestbookJson); Guestbook gb = new Guestbook(); gb = sut.parseGuestbook(jsonObj, gb); @@ -797,4 +797,39 @@ public void testGuestbook() throws JsonParseException { assertEquals("Purple", gb.getCustomQuestions().get(2).getCustomQuestionValues().get(3).getValueString()); assertEquals(3, gb.getCustomQuestions().get(2).getCustomQuestionValues().get(3).getDisplayOrder()); } + + @Test + public void testGuestbookResponse() throws JsonParseException { + JsonObject jsonObj = JsonUtil.getJsonObject(guestbookJson); + Guestbook gb = new Guestbook(); + gb = sut.parseGuestbook(jsonObj, gb); + Long i = 1L; + for (CustomQuestion cq : gb.getCustomQuestions()) { + cq.setId(i++); + } + + final String guestbookResponseJson = """ + { + "answers": [ + { + "id": 1, + "value": "Good" + }, + { + "id": 2, + "value": ["Multi","Line"] + }, + { + "id": 3, + "value": "Yellow" + } + ] + } + """; + + GuestbookResponse guestbookResponse = new GuestbookResponse(); + guestbookResponse.setGuestbook(gb); + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson); + GuestbookResponse gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + } } From 1a1feadbbe4684a8db806176cf19887d66add75e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:17:34 -0500 Subject: [PATCH 03/52] validating response --- .../iq/dataverse/util/json/JsonParser.java | 15 ++++++- src/main/java/propertyFiles/Bundle.properties | 3 ++ .../harvard/iq/dataverse/api/AccessIT.java | 10 ++++- .../harvard/iq/dataverse/api/DatasetsIT.java | 24 +++++++++-- .../dataverse/util/json/JsonParserTest.java | 41 +++++++++++++++++++ 5 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 226cec945bf..804a3c3cbee 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -616,7 +616,7 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons cqr.setCustomQuestion(cq); String response = null; if (cq == null) { - throw new JsonParseException("Guestbook Custom Question ID not found!"); + throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound",List.of(cqId.toString()))); } else if (cq.getQuestionType().equalsIgnoreCase("textarea")) { String lineFeed = String.valueOf((char) 10); JsonArray jsonArray = answer.getJsonArray("value"); @@ -625,7 +625,7 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons } else if (cq.getQuestionType().equalsIgnoreCase("options")) { String option = answer.getString("value"); if (!cq.getCustomQuestionOptions().contains(option)) { - throw new JsonParseException("Guestbook Custom Question Answer not an option!"); + throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseInvalidOption", List.of(option))); } response = option; } else { @@ -633,9 +633,20 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons } cqr.setResponse(response); customQuestionResponses.add(cqr); + cqMap.remove(cqId); // remove so we can check the remaining for missing required questions } guestbookResponse.setCustomQuestionResponses(customQuestionResponses); // verify each required question is in the response + List missingReponses = new ArrayList<>(); + for (Map.Entry e : cqMap.entrySet()) { + if (e.getValue().isRequired()) { + missingReponses.add(e.getValue().getQuestionString()); + } + } + if (!missingReponses.isEmpty()) { + String missing = String.join(",", missingReponses); + throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseMissingRequired", List.of(missing))); + } return guestbookResponse; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 395b6c1e2cf..a1f54ccc01c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2924,6 +2924,9 @@ access.api.requestAccess.failure.requestExists=An access request for this file o access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. access.api.requestAccess.failure.guestbookresponseMissing=You may not request access to this file without the required Guestbook response. +access.api.requestAccess.failure.guestbookresponseMissingRequired=Guestbook Custom Question Answer is required but not present ({0}). +access.api.requestAccess.failure.guestbookresponseInvalidOption=Guestbook Custom Question Answer not a valid option ({0}). +access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound=Guestbook Custom Question ID {0} not found. access.api.requestAccess.noKey=You must provide a key to request access to a file. access.api.requestAccess.fileNotFound=Could not find datafile with id {0}. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index cee896d4938..f8bda4972d3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Guestbook; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import io.restassured.RestAssured; @@ -27,8 +28,7 @@ import java.util.zip.ZipInputStream; import static jakarta.ws.rs.core.Response.Status.*; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.*; @@ -548,6 +548,12 @@ public void testRequestAccess() throws InterruptedException, IOException, JsonPa requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando, guestbookResponseJson); requestFileAccessResponse.prettyPrint(); assertEquals(200, requestFileAccessResponse.getStatusCode()); + // Request a second time should fail since the request was already made + requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando, guestbookResponseJson); + requestFileAccessResponse.prettyPrint(); + requestFileAccessResponse.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.requestExists"))); Response listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiToken); listAccessRequestResponse.prettyPrint(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 97e3494c937..fc44945a8de 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -7502,6 +7502,8 @@ public void testGetDatasetWithGuestbook() throws IOException, JsonParseException @Test public void testGetDatasetWithTermsOfUseAndGuestbook() throws IOException, JsonParseException { + // update ds guestbook + // delete dataset guestbook File guestbookJson = new File("scripts/api/data/guestbook-test.json"); String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); // Create users, Dataverse, and Dataset @@ -7529,9 +7531,25 @@ public void testGetDatasetWithTermsOfUseAndGuestbook() throws IOException, JsonP .statusCode(OK.getStatusCode()) .body("data.message", equalTo(BundleUtil.getStringFromBundle("datasets.api.updateLicense.success"))); - // Publish - UtilIT.publishDataverseViaNativeApi(ownerAlias, adminApiToken).prettyPrint(); - UtilIT.publishDatasetViaNativeApi(persistentId, "major", adminApiToken).prettyPrint(); + // Test update dataset guestbook + Response updateDatasetResponse = UtilIT.updateDatasetGuestbook(persistentId, guestbook.getId(), adminApiToken); + updateDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Response getDatasetResponse = UtilIT.getDatasetVersion(persistentId, ":latest", adminApiToken); + getDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.termsOfUse", equalTo("testTermsOfUse")) + .body("data.guestbookId", equalTo(guestbook.getId().intValue())); + + // Test delete dataset guestbook + updateDatasetResponse = UtilIT.updateDatasetGuestbook(persistentId, null, adminApiToken); + updateDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + getDatasetResponse = UtilIT.getDatasetVersion(persistentId, ":latest", adminApiToken); + getDatasetResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.termsOfUse", equalTo("testTermsOfUse")) + .body("data.guestbookId", nullValue()); } private String getSuperuserToken() { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index 668a7e45995..73451aeeb71 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -806,6 +806,7 @@ public void testGuestbookResponse() throws JsonParseException { Long i = 1L; for (CustomQuestion cq : gb.getCustomQuestions()) { cq.setId(i++); + cq.setRequired(true); } final String guestbookResponseJson = """ @@ -826,10 +827,50 @@ public void testGuestbookResponse() throws JsonParseException { ] } """; + final String guestbookResponseJsonMissing3 = """ + { + "answers": [ + { + "id": 1, + "value": "Good" + }, + { + "id": 2, + "value": ["Multi","Line"] + } + ] + } + """; GuestbookResponse guestbookResponse = new GuestbookResponse(); guestbookResponse.setGuestbook(gb); jsonObj = JsonUtil.getJsonObject(guestbookResponseJson); GuestbookResponse gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + assertTrue(gbr.getCustomQuestionResponses().size() == 3); + + // Test missing required question response + try { + jsonObj = JsonUtil.getJsonObject(guestbookResponseJsonMissing3); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + } catch (JsonParseException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("What color car do you drive")); + } + // Test invalid option in question response + try { + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("Yellow", "Green")); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + } catch (JsonParseException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("not a valid option (Green)")); + } + // Test invalid Custom Question ID in question response + try { + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("3", "4")); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + } catch (JsonParseException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("ID 4 not found")); + } } } From 6d16aae7dee98c8e11edfc6db51f93a5c9b84c90 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:28:55 -0500 Subject: [PATCH 04/52] validating response --- .../harvard/iq/dataverse/api/DatasetsIT.java | 68 ++++--------------- 1 file changed, 15 insertions(+), 53 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index fc44945a8de..f35b1cb1bd6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -7412,7 +7412,7 @@ public void testExcludeEmailOverride() { } @Test - public void testGetDatasetWithGuestbook() throws IOException, JsonParseException { + public void testGetDatasetWithTermsOfUseAndGuestbook() throws IOException, JsonParseException { String apiToken = getSuperuserToken(); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); String ownerAlias = UtilIT.getAliasFromResponse(createDataverseResponse); @@ -7425,6 +7425,19 @@ public void testGetDatasetWithGuestbook() throws IOException, JsonParseException // Create a Guestbook Guestbook guestbook = UtilIT.createRandomGuestbook(ownerAlias, persistentId, apiToken); + // Create a license for Terms of Use + String jsonString = """ + { + "customTerms": { + "termsOfUse": "testTermsOfUse" + } + } + """; + Response updateLicenseResponse = UtilIT.updateLicense(datasetId.toString(), jsonString, apiToken); + updateLicenseResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo(BundleUtil.getStringFromBundle("datasets.api.updateLicense.success"))); + // Enable the Guestbook with invalid enable flag Response guestbookEnableResponse = UtilIT.enableGuestbook(ownerAlias, guestbook.getId(), apiToken, "x"); guestbookEnableResponse.prettyPrint(); @@ -7436,6 +7449,7 @@ public void testGetDatasetWithGuestbook() throws IOException, JsonParseException getDataset.prettyPrint(); getDataset.then().assertThat() .statusCode(OK.getStatusCode()) + .body("data[0].termsOfUse", equalTo("testTermsOfUse")) .body("data[0].guestbookId", equalTo(guestbook.getId().intValue())); getDataset = UtilIT.nativeGet(datasetId, apiToken); @@ -7500,58 +7514,6 @@ public void testGetDatasetWithGuestbook() throws IOException, JsonParseException .body("data.guestbookId", equalTo(guestbook.getId().intValue())); } - @Test - public void testGetDatasetWithTermsOfUseAndGuestbook() throws IOException, JsonParseException { - // update ds guestbook - // delete dataset guestbook - File guestbookJson = new File("scripts/api/data/guestbook-test.json"); - String guestbookAsJson = new String(Files.readAllBytes(Paths.get(guestbookJson.getAbsolutePath()))); - // Create users, Dataverse, and Dataset - String adminApiToken = getSuperuserToken(); - Response createResponse = UtilIT.createRandomUser(); - String apiToken = UtilIT.getApiTokenFromResponse(createResponse); - Response createDataverseResponse = UtilIT.createRandomDataverse(adminApiToken); - String ownerAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(ownerAlias, adminApiToken); - createDatasetResponse.prettyPrint(); - String persistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); - Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); - // Create a Guestbook - Guestbook guestbook = UtilIT.createRandomGuestbook(ownerAlias, persistentId, adminApiToken); - // Create a license for Terms of Use - String jsonString = """ - { - "customTerms": { - "termsOfUse": "testTermsOfUse" - } - } - """; - Response updateLicenseResponse = UtilIT.updateLicense(datasetId.toString(), jsonString, adminApiToken); - updateLicenseResponse.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.message", equalTo(BundleUtil.getStringFromBundle("datasets.api.updateLicense.success"))); - - // Test update dataset guestbook - Response updateDatasetResponse = UtilIT.updateDatasetGuestbook(persistentId, guestbook.getId(), adminApiToken); - updateDatasetResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - Response getDatasetResponse = UtilIT.getDatasetVersion(persistentId, ":latest", adminApiToken); - getDatasetResponse.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.termsOfUse", equalTo("testTermsOfUse")) - .body("data.guestbookId", equalTo(guestbook.getId().intValue())); - - // Test delete dataset guestbook - updateDatasetResponse = UtilIT.updateDatasetGuestbook(persistentId, null, adminApiToken); - updateDatasetResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - getDatasetResponse = UtilIT.getDatasetVersion(persistentId, ":latest", adminApiToken); - getDatasetResponse.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.termsOfUse", equalTo("testTermsOfUse")) - .body("data.guestbookId", nullValue()); - } - private String getSuperuserToken() { Response createResponse = UtilIT.createRandomUser(); String adminApiToken = UtilIT.getApiTokenFromResponse(createResponse); From 411befd0f75bca95bb0b76d36f0a2b32fb3e58a0 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:33:02 -0500 Subject: [PATCH 05/52] Potential fix for code scanning alert no. 354: Information exposure through an error message Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index f64eec31ef7..6472a5c639a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5997,7 +5997,8 @@ public Response updateDatasetGuestbook(@Context ContainerRequestContext crc, @Pa } catch (NumberFormatException nfe) { return error(NOT_FOUND, "Could not find a guestbook with id " + guestbookId); } catch (CommandException ex) { - return error(BAD_REQUEST, ex.getMessage()); + logger.log(Level.WARNING, "Failed to update dataset guestbook for dataset " + dataset.getId(), ex); + return error(BAD_REQUEST, "Failed to update dataset guestbook."); } return ok("Guestbook " + guestbookId + " set"); @@ -6018,7 +6019,8 @@ public Response deleteDatasetGuestbook(@Context ContainerRequestContext crc, @Pa commandEngine.submit(update_cmd); } catch (CommandException ex) { - return error(BAD_REQUEST, ex.getMessage()); + logger.log(Level.WARNING, "Failed to remove dataset guestbook for dataset " + dataset.getId(), ex); + return error(BAD_REQUEST, "Failed to remove dataset guestbook."); } return ok("Guestbook removed " + guestbookId); } else { From 4833dd89aa705d4f3d69d9225fed423fdbb94cdd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:33:22 -0500 Subject: [PATCH 06/52] Potential fix for code scanning alert no. 355: Information exposure through an error message Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 6472a5c639a..3cdc761f069 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -6018,7 +6018,8 @@ public Response deleteDatasetGuestbook(@Context ContainerRequestContext crc, @Pa commandEngine.submit(update_cmd); - } catch (CommandException ex) { + logger.log(Level.WARNING, "Failed to remove guestbook from dataset " + dataset.getId(), ex); + return error(BAD_REQUEST, "Failed to remove guestbook."); logger.log(Level.WARNING, "Failed to remove dataset guestbook for dataset " + dataset.getId(), ex); return error(BAD_REQUEST, "Failed to remove dataset guestbook."); } From bbea198a1e84e42a03f86d95d9664af69d237475 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:33:47 -0500 Subject: [PATCH 07/52] Potential fix for code scanning alert no. 356: Information exposure through an error message Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java index 681fa7abd3d..1f264501816 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java @@ -21,6 +21,7 @@ import java.sql.Timestamp; import java.time.Instant; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; @@ -60,7 +61,8 @@ public Response createGuestbook(@Context ContainerRequestContext crc, @PathParam JsonObject jsonObj = JsonUtil.getJsonObject(jsonBody); jsonParser().parseGuestbook(jsonObj, guestbook); } catch (JsonException | JsonParseException ex) { - return badRequest(ex.getMessage()); + logger.log(Level.WARNING, "Error parsing guestbook JSON", ex); + return badRequest("Error parsing guestbook JSON"); } guestbook.setCreateTime(Timestamp.from(Instant.now())); execCommand(new CreateGuestbookCommand(guestbook, req, dataverse)); From 1a73f06967fb72ab436bde146561bcff8772f2a4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:48:01 -0500 Subject: [PATCH 08/52] code cleanup --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 10 ++-------- .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3cdc761f069..1e72469fa84 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -5989,11 +5989,8 @@ public Response updateDatasetGuestbook(@Context ContainerRequestContext crc, @Pa if (guestbook == null) { return error(NOT_FOUND, "Could not find a guestbook with id " + guestbookId); } - UpdateDatasetGuestbookCommand update_cmd = new UpdateDatasetGuestbookCommand(dataset, guestbook, req); - commandEngine.submit(update_cmd); - } catch (NumberFormatException nfe) { return error(NOT_FOUND, "Could not find a guestbook with id " + guestbookId); } catch (CommandException ex) { @@ -6015,12 +6012,9 @@ public Response deleteDatasetGuestbook(@Context ContainerRequestContext crc, @Pa Long guestbookId = dataset.getGuestbook().getId(); try { UpdateDatasetGuestbookCommand update_cmd = new UpdateDatasetGuestbookCommand(dataset, null, req); - commandEngine.submit(update_cmd); - - logger.log(Level.WARNING, "Failed to remove guestbook from dataset " + dataset.getId(), ex); - return error(BAD_REQUEST, "Failed to remove guestbook."); - logger.log(Level.WARNING, "Failed to remove dataset guestbook for dataset " + dataset.getId(), ex); + } catch (CommandException ex) { + logger.log(Level.WARNING, "Failed to remove dataset guestbook from dataset " + dataset.getId(), ex); return error(BAD_REQUEST, "Failed to remove dataset guestbook."); } return ok("Guestbook removed " + guestbookId); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index f35b1cb1bd6..286cca33936 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -7500,7 +7500,7 @@ public void testGetDatasetWithTermsOfUseAndGuestbook() throws IOException, JsonP setGuestbook.prettyPrint(); setGuestbook.then().assertThat() .statusCode(BAD_REQUEST.getStatusCode()) - .body("message", startsWith("Could not find an available guestbook")); + .body("message", startsWith("Failed to update dataset guestbook")); // Enable the Guestbook. Add it to the Dataset. Then disable it. // Show that the guestbook is still returned in the dataset Json even if it's disabled From b52b8565bf4dcf596f66de18dacf3da7b8f012bb Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:08:01 -0500 Subject: [PATCH 09/52] add -Ddataverse.files.guestbook-at-request=true for testing --- docker/compose/demo/compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 779cf37a931..80f0ea08f5c 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -20,6 +20,7 @@ services: -Ddataverse.files.file1.type=file -Ddataverse.files.file1.label=Filesystem -Ddataverse.files.file1.directory=${STORAGE_DIR}/store + -Ddataverse.files.guestbook-at-request=true -Ddataverse.pid.providers=perma1 -Ddataverse.pid.default-provider=perma1 -Ddataverse.pid.perma1.type=perma From 6bbda525fe71b4887dce9fce543d69b8ab5346c3 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:31:37 -0500 Subject: [PATCH 10/52] fix test --- docker-compose-dev.yml | 2 +- docker/compose/demo/compose.yml | 1 - .../harvard/iq/dataverse/api/AccessIT.java | 107 ++++++++++++++++-- 3 files changed, 98 insertions(+), 12 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 95383ea1670..88b902dfc7f 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -54,13 +54,13 @@ services: -Ddataverse.files.minio1.download-redirect=false -Ddataverse.files.minio1.access-key=4cc355_k3y -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y - -Ddataverse.files.guestbook-at-request=true -Ddataverse.pid.providers=fake -Ddataverse.pid.default-provider=fake -Ddataverse.pid.fake.type=FAKE -Ddataverse.pid.fake.label=FakeDOIProvider -Ddataverse.pid.fake.authority=10.5072 -Ddataverse.pid.fake.shoulder=FK2/ + #-Ddataverse.files.guestbook-at-request=true #-Ddataverse.lang.directory=/dv/lang ports: - "8080:8080" # HTTP (Dataverse Application) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 80f0ea08f5c..779cf37a931 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -20,7 +20,6 @@ services: -Ddataverse.files.file1.type=file -Ddataverse.files.file1.label=Filesystem -Ddataverse.files.file1.directory=${STORAGE_DIR}/store - -Ddataverse.files.guestbook-at-request=true -Ddataverse.pid.providers=perma1 -Ddataverse.pid.default-provider=perma1 -Ddataverse.pid.perma1.type=perma diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index f8bda4972d3..a2f5ff26eac 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -16,6 +16,7 @@ import org.hamcrest.collection.IsMapContaining; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; @@ -487,10 +488,96 @@ private HashMap readZipResponse(InputStream iStrea return fileStreams; } - + @Test - public void testRequestAccess() throws InterruptedException, IOException, JsonParseException { - + public void testRequestAccess() throws InterruptedException { + + String pathToJsonFile = "scripts/api/data/dataset-create-new.json"; + Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetIdNew = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + basicFileName = "004.txt"; + String basicPathToFile = "scripts/search/data/replace_test/" + basicFileName; + Response basicAddResponse = UtilIT.uploadFileViaNative(datasetIdNew.toString(), basicPathToFile, apiToken); + Integer basicFileIdNew = JsonPath.from(basicAddResponse.body().asString()).getInt("data.files[0].dataFile.id"); + + String tabFile3NameRestrictedNew = "stata13-auto-withstrls.dta"; + String tab3PathToFile = "scripts/search/data/tabular/" + tabFile3NameRestrictedNew; + Response tab3AddResponse = UtilIT.uploadFileViaNative(datasetIdNew.toString(), tab3PathToFile, apiToken); + Integer tabFile3IdRestrictedNew = JsonPath.from(tab3AddResponse.body().asString()).getInt("data.files[0].dataFile.id"); + + assertTrue(UtilIT.sleepForLock(datasetIdNew.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if Ingest Lock exceeds max duration " + tab3PathToFile); + + Response restrictResponse = UtilIT.restrictFile(tabFile3IdRestrictedNew.toString(), true, apiToken); + restrictResponse.prettyPrint(); + restrictResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + assertEquals(200, createUser.getStatusCode()); + String apiTokenRando = UtilIT.getApiTokenFromResponse(createUser); + String apiIdentifierRando = UtilIT.getUsernameFromResponse(createUser); + + Response randoDownload = UtilIT.downloadFile(tabFile3IdRestrictedNew, apiTokenRando); + assertEquals(403, randoDownload.getStatusCode()); + + Response requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); + //Cannot request until we set the dataset to allow requests + assertEquals(400, requestFileAccessResponse.getStatusCode()); + //Update Dataset to allow requests + Response allowAccessRequestsResponse = UtilIT.allowAccessRequests(datasetIdNew.toString(), true, apiToken); + assertEquals(200, allowAccessRequestsResponse.getStatusCode()); + //Must republish to get it to work + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetIdNew, "major", apiToken); + assertEquals(200, publishDataset.getStatusCode()); + + requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); + assertEquals(200, requestFileAccessResponse.getStatusCode()); + + Response listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiToken); + listAccessRequestResponse.prettyPrint(); + assertEquals(200, listAccessRequestResponse.getStatusCode()); + System.out.println("List Access Request: " + listAccessRequestResponse.prettyPrint()); + + listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiTokenRando); + listAccessRequestResponse.prettyPrint(); + assertEquals(403, listAccessRequestResponse.getStatusCode()); + + Response rejectFileAccessResponse = UtilIT.rejectFileAccessRequest(tabFile3IdRestrictedNew.toString(), "@" + apiIdentifierRando, apiToken); + assertEquals(200, rejectFileAccessResponse.getStatusCode()); + + requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); + //grant file access + Response grantFileAccessResponse = UtilIT.grantFileAccess(tabFile3IdRestrictedNew.toString(), "@" + apiIdentifierRando, apiToken); + assertEquals(200, grantFileAccessResponse.getStatusCode()); + + //if you make a request while you have been granted access you should get a command exception + requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); + assertEquals(400, requestFileAccessResponse.getStatusCode()); + + //if you make a request of a public file you should also get a command exception + requestFileAccessResponse = UtilIT.requestFileAccess(basicFileIdNew.toString(), apiTokenRando); + assertEquals(400, requestFileAccessResponse.getStatusCode()); + + + //Now should be able to download + randoDownload = UtilIT.downloadFile(tabFile3IdRestrictedNew, apiTokenRando); + assertEquals(OK.getStatusCode(), randoDownload.getStatusCode()); + + //revokeFileAccess + Response revokeFileAccessResponse = UtilIT.revokeFileAccess(tabFile3IdRestrictedNew.toString(), "@" + apiIdentifierRando, apiToken); + assertEquals(200, revokeFileAccessResponse.getStatusCode()); + + listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiToken); + assertEquals(404, listAccessRequestResponse.getStatusCode()); + } + + @Test + @Disabled // Only run manually after setting JVM setting -Ddataverse.files.guestbook-at-request=true + public void testRequestAccessWithGuestbook() throws IOException, JsonParseException { + String pathToJsonFile = "scripts/api/data/dataset-create-new.json"; Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); createDatasetResponse.prettyPrint(); @@ -500,7 +587,7 @@ public void testRequestAccess() throws InterruptedException, IOException, JsonPa // Create a Guestbook Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, apiToken); String guestbookResponseJson = UtilIT.generateGuestbookResponse(guestbook); - + basicFileName = "004.txt"; String basicPathToFile = "scripts/search/data/replace_test/" + basicFileName; Response basicAddResponse = UtilIT.uploadFileViaNative(datasetIdNew.toString(), basicPathToFile, apiToken); @@ -512,7 +599,7 @@ public void testRequestAccess() throws InterruptedException, IOException, JsonPa Integer tabFile3IdRestrictedNew = JsonPath.from(tab3AddResponse.body().asString()).getInt("data.files[0].dataFile.id"); assertTrue(UtilIT.sleepForLock(datasetIdNew.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if Ingest Lock exceeds max duration " + tab3PathToFile); - + Response restrictResponse = UtilIT.restrictFile(tabFile3IdRestrictedNew.toString(), true, apiToken); restrictResponse.prettyPrint(); restrictResponse.then().assertThat() @@ -539,8 +626,9 @@ public void testRequestAccess() throws InterruptedException, IOException, JsonPa // Set the guestbook on the Dataset UtilIT.updateDatasetGuestbook(persistentIdNew, guestbook.getId(), apiToken).prettyPrint(); - // Request file access WITHOUT the required Guestbook Response (getEffectiveGuestbookEntryAtRequest) + // Set the response required on the Access Request as apposed to being on Download UtilIT.setGuestbookEntryOnRequest(datasetId.toString(), apiToken, Boolean.TRUE).prettyPrint(); + // Request file access WITHOUT the required Guestbook Response (getEffectiveGuestbookEntryAtRequest) requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); requestFileAccessResponse.prettyPrint(); assertEquals(400, requestFileAccessResponse.getStatusCode()); @@ -571,21 +659,20 @@ public void testRequestAccess() throws InterruptedException, IOException, JsonPa //grant file access Response grantFileAccessResponse = UtilIT.grantFileAccess(tabFile3IdRestrictedNew.toString(), "@" + apiIdentifierRando, apiToken); assertEquals(200, grantFileAccessResponse.getStatusCode()); - + //if you make a request while you have been granted access you should get a command exception requestFileAccessResponse = UtilIT.requestFileAccess(tabFile3IdRestrictedNew.toString(), apiTokenRando); assertEquals(400, requestFileAccessResponse.getStatusCode()); - + //if you make a request of a public file you should also get a command exception requestFileAccessResponse = UtilIT.requestFileAccess(basicFileIdNew.toString(), apiTokenRando); assertEquals(400, requestFileAccessResponse.getStatusCode()); - //Now should be able to download randoDownload = UtilIT.downloadFile(tabFile3IdRestrictedNew, apiTokenRando); assertEquals(OK.getStatusCode(), randoDownload.getStatusCode()); - //revokeFileAccess + //revokeFileAccess Response revokeFileAccessResponse = UtilIT.revokeFileAccess(tabFile3IdRestrictedNew.toString(), "@" + apiIdentifierRando, apiToken); assertEquals(200, revokeFileAccessResponse.getStatusCode()); From a630c9c63a99b6472dd0a7e7eb4881537cc61263 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 6 Feb 2026 15:55:10 -0500 Subject: [PATCH 11/52] adding post for download datafile with guestbook response --- .../edu/harvard/iq/dataverse/api/Access.java | 146 ++++++++++++---- .../WebApplicationExceptionHandler.java | 5 +- src/main/java/propertyFiles/Bundle.properties | 4 +- .../edu/harvard/iq/dataverse/api/FilesIT.java | 156 ++++++++++++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 8 + 5 files changed, 253 insertions(+), 66 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index eb9dbd9abdc..c8cfb097180 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; @@ -27,16 +28,16 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.FileUtil; -import edu.harvard.iq.dataverse.util.StringUtil; -import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.*; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import jakarta.ejb.EJB; import jakarta.inject.Inject; -import jakarta.json.*; +import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; import jakarta.persistence.TypedQuery; import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.*; @@ -363,8 +364,70 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI } return Response.ok(downloadInstance).build(); } - - + + @POST + @AuthRequired + @Path("datafile/{fileId:.+}") + @Produces({"application/xml","*/*"}) + public Response datafileWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) { + + // check first if there's a trailing slash, and chop it: + while (fileId.lastIndexOf('/') == fileId.length() - 1) { + fileId = fileId.substring(0, fileId.length() - 1); + } + + if (fileId.indexOf('/') > -1) { + // This is for embedding folder names into the Access API URLs; + // something like /api/access/datafile/folder/subfolder/1234 + // instead of the normal /api/access/datafile/1234 notation. + // this is supported only for recreating folders during recursive downloads - + // i.e. they are embedded into the URL for the remote client like wget, + // but can be safely ignored here. + fileId = fileId.substring(fileId.lastIndexOf('/') + 1); + } + + DataFile df = findDataFileOrDieWrapper(fileId); + GuestbookResponse gbr = null; + + if (df.isHarvested()) { + String errorMessage = "Datafile " + fileId + " is a harvested file that cannot be accessed in this Dataverse"; + throw new NotFoundException(errorMessage); + // (nobody should ever be using this API on a harvested DataFile)! + } + + // This will throw a ForbiddenException if access isn't authorized: + checkAuthorization(crc, df); + + AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); + try { + if (checkGuestbookRequiredResponse(crc, df)) { + gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, jsonBody, user); + if (gbr != null) { + engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); + } else { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + } + } else if (gbrecs != true && df.isReleased()) { + // Write Guestbook record if not done previously and file is released + gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, user); + } + } catch (JsonParseException | CommandException ex) { + List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); + } + + String baseUrl = uriInfo.getAbsolutePath().toString() + "?gbrecs=true"; + String key = ""; + ApiToken apiToken = authSvc.findApiTokenByUser(user); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { + key = apiToken.getTokenString(); + } + String signedUrl = UrlSignerUtil.signUrl(baseUrl, 10, user.getUserIdentifier(), "GET", key); + + return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); + } + /* * Variants of the Access API calls for retrieving datafile-level * Metadata. @@ -1295,7 +1358,7 @@ public Response deleteAuxiliaryFileWithVersion(@Context ContainerRequestContext } catch (FileNotFoundException e) { throw new NotFoundException(); } catch(IOException io) { - throw new ServerErrorException("IO Exception trying remove auxiliary file", Response.Status.INTERNAL_SERVER_ERROR, io); + throw new ServerErrorException("IO Exception trying remove auxiliary file", INTERNAL_SERVER_ERROR, io); } return ok("Auxiliary file deleted."); @@ -1363,7 +1426,6 @@ public Response requestFileAccess(@Context ContainerRequestContext crc DataverseRequest dataverseRequest; DataFile dataFile; - try { dataFile = findDataFileOrDie(fileToRequestAccessId); } catch (WrappedResponse ex) { @@ -1397,33 +1459,22 @@ public Response requestFileAccess(@Context ContainerRequestContext crc return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.requestExists")); } - // Is Guestbook response required? - // The response will be true (guestbook displays when making a request), false (guestbook displays at download), or will indicate that the dataset inherits one of these settings. - GuestbookResponse guestbookResponse = null; - if (dataFile.getOwner().getEffectiveGuestbookEntryAtRequest()) { + try { + // Is Guestbook response required? + // getEffectiveGuestbookEntryAtRequest response will be true (guestbook displays when making a request), false (guestbook displays at download), or will indicate that the dataset inherits one of these settings. + // Even if it is not required we will take it if it's included. Dataset must have a guestbook that is enabled Dataset ds = dataFile.getOwner(); + GuestbookResponse guestbookResponse = getGuestbookResponseFromBody(dataFile, GuestbookResponse.ACCESS_REQUEST, jsonBody, getRequestUser(crc)); if (ds.getGuestbook() != null && ds.getGuestbook().isEnabled()) { - // response is required - try { - if (jsonBody == null || jsonBody.isBlank()) { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseMissing")); - } - JsonObject jsonObj = JsonUtil.getJsonObject(jsonBody).getJsonObject("guestbookResponse"); - guestbookResponse = guestbookResponseService.initAPIGuestbookResponse(ds, dataFile, null, requestor); - guestbookResponse.setEventType(GuestbookResponse.ACCESS_REQUEST); - // Parse custom question answers - jsonParser().parseGuestbookResponse(jsonObj, guestbookResponse); + if (ds.getEffectiveGuestbookEntryAtRequest() && guestbookResponse == null) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookAccessRequestResponseMissing")); + } else if (guestbookResponse != null) { engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset())); - } catch (JsonException | JsonParseException | CommandException ex) { - List args = Arrays.asList(dataFile.getDisplayName(), ex.getLocalizedMessage()); - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.commandError", args)); } } - } - try { engineSvc.submit(new RequestAccessCommand(dataverseRequest, dataFile, guestbookResponse, true)); - } catch (CommandException ex) { + } catch (CommandException | JsonParseException ex) { List args = Arrays.asList(dataFile.getDisplayName(), ex.getLocalizedMessage()); return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.commandError", args)); } @@ -1472,7 +1523,7 @@ public Response listFileAccessRequests(@Context ContainerRequestContext crc, @Pa if (requests == null || requests.isEmpty()) { List args = Arrays.asList(dataFile.getDisplayName()); - return error(Response.Status.NOT_FOUND, BundleUtil.getStringFromBundle("access.api.requestList.noRequestsFound", args)); + return error(NOT_FOUND, BundleUtil.getStringFromBundle("access.api.requestList.noRequestsFound", args)); } JsonArrayBuilder userArray = Json.createArrayBuilder(); @@ -1713,6 +1764,41 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @ return ok(jsonObjectBuilder); } + private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df) throws WebApplicationException { + // Check if guestbook response is required + if (df.isRestricted() && df.getOwner().hasEnabledGuestbook() && getRequestUser(crc) instanceof AuthenticatedUser) { + AuthenticatedUser user = (AuthenticatedUser)getRequestUser(crc); + List gbrList = guestbookResponseService.findByAuthenticatedUserId(user); + boolean responseFound = false; + if (gbrList != null) { + // find a matching response + for (GuestbookResponse r : gbrList) { + if (r.getDataFile().getId() == df.getId()) { + responseFound = true; + break; + } + } + } + return !responseFound; // if we find a response then it is not required to add another one + } + return false; + } + + private GuestbookResponse getGuestbookResponseFromBody(DataFile dataFile, String type, String jsonBody, User requestor) throws JsonParseException { + Dataset ds = dataFile.getOwner(); + GuestbookResponse guestbookResponse = null; + + if (jsonBody != null && !jsonBody.isBlank()) { + JsonObject guestbookResponseObj = JsonUtil.getJsonObject(jsonBody).getJsonObject("guestbookResponse"); + guestbookResponse = guestbookResponseService.initAPIGuestbookResponse(ds, dataFile, null, requestor); + guestbookResponse.setEventType(type); + // Parse custom question answers + jsonParser().parseGuestbookResponse(guestbookResponseObj, guestbookResponse); + } + + return guestbookResponse; + } + // checkAuthorization is a convenience method; it calls the boolean method // isAccessAuthorized(), the actual workhorse, and throws a 403 exception if not. private void checkAuthorization(ContainerRequestContext crc, DataFile df) throws WebApplicationException { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index af9aeffa1c9..142e595db5d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -7,19 +7,18 @@ import edu.harvard.iq.dataverse.api.util.JsonResponseBuilder; import edu.harvard.iq.dataverse.util.BundleUtil; - import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; import jakarta.ws.rs.ext.Provider; +import org.apache.commons.lang3.StringUtils; + import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.lang3.StringUtils; - /** * Catches all types of web application exceptions like NotFoundException, etc etc and handles them properly. */ diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index a1f54ccc01c..f324af3762b 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2923,7 +2923,7 @@ access.api.requestAccess.failure.commandError=Problem trying request access on { access.api.requestAccess.failure.requestExists=An access request for this file on your behalf already exists. access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. -access.api.requestAccess.failure.guestbookresponseMissing=You may not request access to this file without the required Guestbook response. +access.api.requestAccess.failure.guestbookAccessRequestResponseMissing=You may not request access to this file without the required Guestbook response. access.api.requestAccess.failure.guestbookresponseMissingRequired=Guestbook Custom Question Answer is required but not present ({0}). access.api.requestAccess.failure.guestbookresponseInvalidOption=Guestbook Custom Question Answer not a valid option ({0}). access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound=Guestbook Custom Question ID {0} not found. @@ -2948,6 +2948,8 @@ access.api.exception.metadata.not.available.for.nontabular.file=This type of met access.api.exception.metadata.restricted.no.permission=You do not have permission to download this file. access.api.exception.version.not.found=Could not find requested dataset version. access.api.exception.dataset.not.found=Could not find requested dataset. +access.api.download.failure.guestbookResponseMissing=You may not download this file without the required Guestbook response. +access.api.download.failure.guestbook.commandError=Problem trying download with guestbook response on {0} : {1} #permission permission.AddDataverse.label=AddDataverse diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 262f3252f9d..4463e09071b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1,60 +1,59 @@ package edu.harvard.iq.dataverse.api; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.Guestbook; +import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; +import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; import io.restassured.RestAssured; +import io.restassured.path.json.JsonPath; +import io.restassured.path.xml.XmlPath; import io.restassured.response.Response; - -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.logging.Logger; - -import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; +import jakarta.json.Json; import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; import jakarta.ws.rs.core.Response.Status; import org.assertj.core.util.Lists; +import org.hamcrest.CoreMatchers; import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.BeforeAll; -import io.restassured.path.json.JsonPath; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; -import static edu.harvard.iq.dataverse.api.ApiConstants.*; -import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; -import static io.restassured.path.json.JsonPath.with; -import io.restassured.path.xml.XmlPath; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.FileUtil; -import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.File; import java.io.IOException; - -import static java.lang.Thread.sleep; - +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.text.MessageFormat; - -import jakarta.json.Json; -import jakarta.json.JsonObjectBuilder; - -import static jakarta.ws.rs.core.Response.Status.*; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.text.MessageFormat; import java.time.Year; -import org.hamcrest.CoreMatchers; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import static edu.harvard.iq.dataverse.api.ApiConstants.*; +import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key; +import static io.restassured.RestAssured.get; +import static io.restassured.path.json.JsonPath.with; +import static jakarta.ws.rs.core.Response.Status.*; +import static java.lang.Thread.sleep; import static org.hamcrest.CoreMatchers.*; import static org.junit.jupiter.api.Assertions.*; @@ -3870,4 +3869,97 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() throws InterruptedExcepti .body("message", equalTo(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date",Collections.singletonList("bad-date")))) .statusCode(BAD_REQUEST.getStatusCode()); } + + @Test + public void testDownloadFileWithGuestbookResponse() throws IOException, JsonParseException { + msgt("testDownloadFileWithGuestbookResponse"); + // Create super user + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + String superusername = UtilIT.getUsernameFromResponse(createUserResponse); + UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200); + + // Create Dataverse + String dataverseAlias = createDataverseGetAlias(apiToken); + + // Create user with no permission + createUserResponse = UtilIT.createRandomUser(); + assertEquals(200, createUserResponse.getStatusCode()); + String apiTokenRando = UtilIT.getApiTokenFromResponse(createUserResponse); + String username = UtilIT.getUsernameFromResponse(createUserResponse); + + // Create Dataset + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String persistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + Response getDatasetMetadata = UtilIT.nativeGet(datasetId, apiToken); + getDatasetMetadata.prettyPrint(); + getDatasetMetadata.then().assertThat().statusCode(200); + + // Create a Guestbook + Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, apiToken); + + // Upload file + String pathToFile1 = "src/main/webapp/resources/images/dataverseproject.png"; + JsonObjectBuilder json1 = Json.createObjectBuilder() + .add("description", "my description1") + .add("directoryLabel", "data/subdir1") + .add("categories", Json.createArrayBuilder().add("Data")); + Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile1, json1.build(), apiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + // Restrict file + Response restrictResponse = UtilIT.restrictFile(fileId.toString(), true, apiToken); + restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); + // Update Dataset to allow requests + Response allowAccessRequestsResponse = UtilIT.allowAccessRequests(datasetId.toString(), true, apiToken); + assertEquals(200, allowAccessRequestsResponse.getStatusCode()); + // Publish dataverse and dataset + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + assertEquals(200, publishDataset.getStatusCode()); + + // Request access + Response requestFileAccessResponse = UtilIT.requestFileAccess(fileId.toString(), apiTokenRando, null); + requestFileAccessResponse.prettyPrint(); + assertEquals(200, requestFileAccessResponse.getStatusCode()); + + // Grant file access + Response grantFileAccessResponse = UtilIT.grantFileAccess(fileId.toString(), "@" + username, apiToken); + grantFileAccessResponse.prettyPrint(); + assertEquals(200, grantFileAccessResponse.getStatusCode()); + + String guestbookResponse = UtilIT.generateGuestbookResponse(guestbook); + + // Get Download Url attempt - Guestbook Response is required but not found + Response downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId, apiTokenRando, null); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .body("status", equalTo(ApiConstants.STATUS_ERROR)) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) + .statusCode(BAD_REQUEST.getStatusCode()); + + // Get Download Url with guestbook response + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId, apiTokenRando, guestbookResponse); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + String signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); + + // Download the file using the signed url + Response signedUrlResponse = get(signedUrl); + signedUrlResponse.prettyPrint(); + assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); + + // Download again with guestbook response already given + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId, apiTokenRando, null); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + signedUrlResponse = get(signedUrl); + assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 38ae66d1f58..d22b902d741 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1246,6 +1246,14 @@ static Response downloadFileOriginal(Integer fileId, String apiToken) { return given() .get("/api/access/datafile/" + fileId + "?format=original&key=" + apiToken); } + static Response getDownloadFileUrlWithGuestbookResponse(Integer fileId, String apiToken, String body) { + RequestSpecification requestSpecification = given(); + requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); + if (body != null) { + requestSpecification.body(body); + } + return requestSpecification.post("/api/access/datafile/" + fileId); + } static Response downloadFiles(Integer[] fileIds) { String getString = "/api/access/datafiles/"; From 9d68d33e5093d5fd7fb19c7f3fc03ff932cf92fd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 6 Feb 2026 16:14:48 -0500 Subject: [PATCH 12/52] fix --- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 2 +- .../api/errorhandlers/WebApplicationExceptionHandler.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index c8cfb097180..573e2ff79f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -368,7 +368,7 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI @POST @AuthRequired @Path("datafile/{fileId:.+}") - @Produces({"application/xml","*/*"}) + @Produces({"application/json"}) public Response datafileWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index 142e595db5d..af9aeffa1c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -7,18 +7,19 @@ import edu.harvard.iq.dataverse.api.util.JsonResponseBuilder; import edu.harvard.iq.dataverse.util.BundleUtil; + import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; import jakarta.ws.rs.ext.Provider; -import org.apache.commons.lang3.StringUtils; - import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.lang3.StringUtils; + /** * Catches all types of web application exceptions like NotFoundException, etc etc and handles them properly. */ From 79c3eaa1c868bb9f60a4ae1196dee08e0ac8898c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Feb 2026 09:40:03 -0500 Subject: [PATCH 13/52] add release note --- .../12001-api-support-termofuse-guestbook.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/release-notes/12001-api-support-termofuse-guestbook.md diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md new file mode 100644 index 00000000000..d38c056b8a6 --- /dev/null +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -0,0 +1,14 @@ +## Feature Request: API to support Download Terms of Use and Guestbook + +## New Endpoint to download a file that required a Guestbook response: POST `/api/access/datafile/{id}` +A post to this endpoint with the body containing a JSON Guestbook Response will save the response and return a signed URL to download the file + +## New CRUD Endpoints for Guestbook: +Create a Guestbook: POST `/api/guestbooks/{dataverseIdentifier}` +Get a Guestbook: POST `/api/guestbooks/{dataverseIdentifier}` +Enable/Disable a Guestbook: PUT `/api/guestbooks/{dataverseIdentifier}/{id}/enabled` Body: `true` or `false` +Note: There is no Update or Delete at this time. You can disable a Guestbook and create a new one. + +## For Guestbook At Request: +When JVM setting -Ddataverse.files.guestbook-at-request=true is used a request for access may require a Guestbook response. +PUT `/api/access/datafile/{id}/requestAccess` will now take a JSON Guestbook response in the body. From b089324afce284cdb395a78f8a7aa1684c89df1e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Feb 2026 14:19:34 -0500 Subject: [PATCH 14/52] new api and updated docs --- .../12001-api-support-termofuse-guestbook.md | 3 +- doc/sphinx-guides/source/api/native-api.rst | 87 +++++++++++++++++++ .../edu/harvard/iq/dataverse/Guestbook.java | 27 +++--- .../iq/dataverse/GuestbookServiceBean.java | 15 +++- .../harvard/iq/dataverse/api/Guestbooks.java | 41 ++++++++- .../edu/harvard/iq/dataverse/api/FilesIT.java | 21 +++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++ 7 files changed, 166 insertions(+), 34 deletions(-) diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md index d38c056b8a6..9fcdcb9ccd3 100644 --- a/doc/release-notes/12001-api-support-termofuse-guestbook.md +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -5,7 +5,8 @@ A post to this endpoint with the body containing a JSON Guestbook Response will ## New CRUD Endpoints for Guestbook: Create a Guestbook: POST `/api/guestbooks/{dataverseIdentifier}` -Get a Guestbook: POST `/api/guestbooks/{dataverseIdentifier}` +Get a Guestbook: GET `/api/guestbooks/{id}` +Get a list of Guestbooks linked to a Dataverse Collection: GET `/api/guestbooks/{dataverseIdentifier}/list` Enable/Disable a Guestbook: PUT `/api/guestbooks/{dataverseIdentifier}/{id}/enabled` Body: `true` or `false` Note: There is no Update or Delete at this time. You can disable a Guestbook and create a new one. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index eab71f8623b..59a9aab3e49 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1168,6 +1168,93 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1" -o myResponses.csv +.. _guestbook-api: + +Create a Guestbook for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + +Create a Guestbook that can be selected for a Dataset. +You must have "EditDataverse" permission on the Dataverse collection. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + export JSON='{"name": "my test guestbook","enabled": true,"emailRequired": true,"nameRequired": true,"institutionRequired": false,"positionRequired": false,"customQuestions": [{"question": "how is your day","required": true,"displayOrder": 0,"type": "text","hidden": false}]}' + + curl -POST -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/guestbooks/{ID}" -d "$JSON" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -POST -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/root" -d '{"name": "my test guestbook","enabled": true,"emailRequired": true,"nameRequired": true,"institutionRequired": false,"positionRequired": false}' + +Get a list of Guestbooks for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + +Get a list of Guestbooks for a Dataverse Collection +You must have "EditDataverse" permission on the Dataverse collection. + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/guestbooks/{ID}/list"` + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/root/list" + +Get a Guestbook for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + +Get a Guestbook by it's id + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1234 + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/guestbooks/{ID}"` + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/1234" + +Enable or Disable a Guestbook for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + +Use this endpoint to enable or disable the Guestbook. A Guestbook can not be deleted or modified since there may be responses linked to it. +You must have "EditDataverse" permission on the Dataverse collection. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export dataverseIdentifier=root + export ID=1234 + + curl -X PUT -d 'true' -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/guestbooks/{dataverseIdentifier}/{ID}/enabled" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X PUT -d 'true' -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/root/1234" + .. _collection-attributes-api: Change Collection Attributes diff --git a/src/main/java/edu/harvard/iq/dataverse/Guestbook.java b/src/main/java/edu/harvard/iq/dataverse/Guestbook.java index 2ef23d1f925..12b81e58506 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Guestbook.java +++ b/src/main/java/edu/harvard/iq/dataverse/Guestbook.java @@ -2,35 +2,28 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.DateUtil; +import jakarta.persistence.*; +import org.hibernate.validator.constraints.NotBlank; + import java.io.Serializable; import java.util.ArrayList; import java.util.Date; -import jakarta.persistence.CascadeType; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.OneToMany; import java.util.List; import java.util.Objects; -import jakarta.persistence.Column; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.OrderBy; -import jakarta.persistence.Temporal; -import jakarta.persistence.TemporalType; -import jakarta.persistence.Transient; - -import edu.harvard.iq.dataverse.util.DateUtil; -import org.hibernate.validator.constraints.NotBlank; /** * * @author skraffmiller */ @Entity +@NamedQueries( + @NamedQuery(name = "Guestbook.findByDataverse", + query = "SELECT gb FROM Guestbook gb WHERE gb.dataverse=:dataverse") +) + public class Guestbook implements Serializable { - + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookServiceBean.java index fcd4e91d455..fc7f361b8b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookServiceBean.java @@ -11,6 +11,8 @@ import jakarta.persistence.PersistenceContext; import jakarta.persistence.Query; +import java.util.List; + /** * * @author skraffmiller @@ -21,8 +23,17 @@ public class GuestbookServiceBean implements java.io.Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; - - + + public List findGuestbooksForGivenDataverse(Dataverse dataverse) { + if (dataverse != null) { + Query query = em.createNamedQuery("Guestbook.findByDataverse"); + query.setParameter("dataverse", dataverse); + return query.getResultList(); + } else { + return List.of(); + } + } + public Long findCountUsages(Long guestbookId, Long dataverseId) { String queryString = ""; if (guestbookId != null && dataverseId != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java index 1f264501816..0bf08015edb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java @@ -4,15 +4,15 @@ import edu.harvard.iq.dataverse.Guestbook; import edu.harvard.iq.dataverse.GuestbookServiceBean; import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateGuestbookCommand; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; import jakarta.ejb.EJB; -import jakarta.json.JsonException; -import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; +import jakarta.json.*; import jakarta.ws.rs.*; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; @@ -49,12 +49,41 @@ public Response getGuestbook(@Context ContainerRequestContext crc, @PathParam("i }, getRequestUser(crc)); } + @GET + @AuthRequired + @Path("{identifier}/list") + public Response getGuestbooks(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier) { + return response( req -> { + Dataverse dataverse = findDataverseOrDie(identifier); + if (permissionSvc.request(req) + .on(dataverse) + .has(Permission.EditDataverse)) { + } else { + return error(Response.Status.FORBIDDEN, "Not authorized"); + } + List guestbooks = guestbookService.findGuestbooksForGivenDataverse(dataverse); + JsonArrayBuilder guestbookArray = Json.createArrayBuilder(); + JsonPrinter jsonPrinter = new JsonPrinter(); + for (Guestbook gb : guestbooks) { + guestbookArray.add(jsonPrinter.json(gb)); + } + return ok(guestbookArray); + }, getRequestUser(crc)); + } + @POST @AuthRequired @Path("{identifier}") public Response createGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String jsonBody) { return response(req -> { Dataverse dataverse = findDataverseOrDie(identifier); + if (permissionSvc.request(req) + .on(dataverse) + .has(Permission.EditDataverse)) { + } else { + return error(Response.Status.FORBIDDEN, "Not authorized"); + } + Guestbook guestbook = new Guestbook(); guestbook.setDataverse(dataverse); try { @@ -87,6 +116,12 @@ public Response enableGuestbook(@Context ContainerRequestContext crc, @PathParam boolean enabled = Util.isTrue(body); return response( req -> { Dataverse dataverse = findDataverseOrDie(identifier); + if (permissionSvc.request(req) + .on(dataverse) + .has(Permission.EditDataverse)) { + } else { + return error(Response.Status.FORBIDDEN, "Not authorized"); + } List guestbooks = dataverse.getGuestbooks(); if (guestbooks != null) { for (Guestbook guestbook : guestbooks) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 4463e09071b..54930a0953c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3873,7 +3873,7 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() throws InterruptedExcepti @Test public void testDownloadFileWithGuestbookResponse() throws IOException, JsonParseException { msgt("testDownloadFileWithGuestbookResponse"); - // Create super user + // Create superuser Response createUserResponse = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); String superusername = UtilIT.getUsernameFromResponse(createUserResponse); @@ -3894,12 +3894,20 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); String persistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); Response getDatasetMetadata = UtilIT.nativeGet(datasetId, apiToken); - getDatasetMetadata.prettyPrint(); getDatasetMetadata.then().assertThat().statusCode(200); + Response getGuestbooksResponse = UtilIT.getGuestbooks(dataverseAlias, apiToken); + getGuestbooksResponse.then().assertThat().statusCode(200); + assertTrue(getGuestbooksResponse.getBody().jsonPath().getList("data").isEmpty()); + // Create a Guestbook Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, apiToken); + // Get the list of Guestbooks + getGuestbooksResponse = UtilIT.getGuestbooks(dataverseAlias, apiToken); + getGuestbooksResponse.then().assertThat().statusCode(200); + assertEquals(1, getGuestbooksResponse.getBody().jsonPath().getList("data").size()); + // Upload file String pathToFile1 = "src/main/webapp/resources/images/dataverseproject.png"; JsonObjectBuilder json1 = Json.createObjectBuilder() @@ -3952,14 +3960,5 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars Response signedUrlResponse = get(signedUrl); signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); - - // Download again with guestbook response already given - downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId, apiTokenRando, null); - downloadResponse.prettyPrint(); - downloadResponse.then().assertThat() - .statusCode(OK.getStatusCode()); - - signedUrlResponse = get(signedUrl); - assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index d22b902d741..72397d87c5f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -609,6 +609,12 @@ static Response getGuestbook(Long guestbookId, String apiToken) { return requestSpec.get("/api/guestbooks/" + guestbookId ); } + static Response getGuestbooks(String dataverseAlias, String apiToken) { + RequestSpecification requestSpec = given() + .header(API_TOKEN_HTTP_HEADER, apiToken); + return requestSpec.get("/api/guestbooks/" + dataverseAlias + "/list" ); + } + static Response enableGuestbook(String dataverseAlias, Long guestbookId, String apiToken, String enable) { Response createGuestbookResponse = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From 7ab9c0a109289c4bad24ee941498ee8b7005ad98 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Feb 2026 15:21:06 -0500 Subject: [PATCH 15/52] updated docs --- doc/sphinx-guides/source/api/dataaccess.rst | 9 ++++++++- doc/sphinx-guides/source/api/native-api.rst | 15 +++++++++++---- .../edu/harvard/iq/dataverse/api/Guestbooks.java | 15 +++------------ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 0782665776d..036b7920b8b 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -91,6 +91,11 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB +.. note:: Restricted files that require a Guestbook response will require an additional step to supply the response. A POST to the same endpoint with the Guestbook Response in the body will return a signed url that can be used to download the file. + + Example :: + + POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -d '{"guestbookResponse": {"answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' Parameters: ~~~~~~~~~~~ @@ -361,7 +366,9 @@ This method requests access to the datafile whose id is passed on the behalf of A curl example using an ``id``:: curl -H "X-Dataverse-key:$API_TOKEN" -X PUT http://$SERVER/api/access/datafile/{id}/requestAccess - + +.. note:: Some installations of Dataverse may require you to provide a Guestbook response when requesting access to certain restricted files. The response can be passed in the body of this call. See "Get a Guestbook for a Dataverse Collection" in the :doc:`native-api`. + Grant File Access: ~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 59a9aab3e49..be966ab545b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1170,8 +1170,11 @@ The fully expanded example above (without environment variables) looks like this .. _guestbook-api: +Guestbooks +~~~~~~~~~~ + Create a Guestbook for a Dataverse Collection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. @@ -1194,13 +1197,15 @@ The fully expanded example above (without environment variables) looks like this curl -POST -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/root" -d '{"name": "my test guestbook","enabled": true,"emailRequired": true,"nameRequired": true,"institutionRequired": false,"positionRequired": false}' Get a list of Guestbooks for a Dataverse Collection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. Get a list of Guestbooks for a Dataverse Collection You must have "EditDataverse" permission on the Dataverse collection. +.. code-block:: bash + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export ID=root @@ -1214,12 +1219,14 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/root/list" Get a Guestbook for a Dataverse Collection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. Get a Guestbook by it's id +.. code-block:: bash + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export ID=1234 @@ -1233,7 +1240,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/guestbooks/1234" Enable or Disable a Guestbook for a Dataverse Collection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java index 0bf08015edb..381f213f54b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Guestbooks.java @@ -55,10 +55,7 @@ public Response getGuestbook(@Context ContainerRequestContext crc, @PathParam("i public Response getGuestbooks(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier) { return response( req -> { Dataverse dataverse = findDataverseOrDie(identifier); - if (permissionSvc.request(req) - .on(dataverse) - .has(Permission.EditDataverse)) { - } else { + if (!permissionSvc.request(req).on(dataverse).has(Permission.EditDataverse)) { return error(Response.Status.FORBIDDEN, "Not authorized"); } List guestbooks = guestbookService.findGuestbooksForGivenDataverse(dataverse); @@ -77,10 +74,7 @@ public Response getGuestbooks(@Context ContainerRequestContext crc, @PathParam(" public Response createGuestbook(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String jsonBody) { return response(req -> { Dataverse dataverse = findDataverseOrDie(identifier); - if (permissionSvc.request(req) - .on(dataverse) - .has(Permission.EditDataverse)) { - } else { + if (!permissionSvc.request(req).on(dataverse).has(Permission.EditDataverse)) { return error(Response.Status.FORBIDDEN, "Not authorized"); } @@ -116,10 +110,7 @@ public Response enableGuestbook(@Context ContainerRequestContext crc, @PathParam boolean enabled = Util.isTrue(body); return response( req -> { Dataverse dataverse = findDataverseOrDie(identifier); - if (permissionSvc.request(req) - .on(dataverse) - .has(Permission.EditDataverse)) { - } else { + if (!permissionSvc.request(req).on(dataverse).has(Permission.EditDataverse)) { return error(Response.Status.FORBIDDEN, "Not authorized"); } List guestbooks = dataverse.getGuestbooks(); From edf0a6ec720e2ea0b90b4a76f99762670039c892 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Feb 2026 15:58:19 -0500 Subject: [PATCH 16/52] refactor and add gb response checks to all download apis --- .../12001-api-support-termofuse-guestbook.md | 16 +- .../edu/harvard/iq/dataverse/api/Access.java | 638 +++++++++++------- src/main/java/propertyFiles/Bundle.properties | 1 + .../edu/harvard/iq/dataverse/api/FilesIT.java | 93 ++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 32 +- 5 files changed, 506 insertions(+), 274 deletions(-) diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md index 9fcdcb9ccd3..ca0600eb6d0 100644 --- a/doc/release-notes/12001-api-support-termofuse-guestbook.md +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -1,7 +1,19 @@ ## Feature Request: API to support Download Terms of Use and Guestbook -## New Endpoint to download a file that required a Guestbook response: POST `/api/access/datafile/{id}` -A post to this endpoint with the body containing a JSON Guestbook Response will save the response and return a signed URL to download the file +## New Endpoints to download a file or files that required a Guestbook response: POST +A post to these endpoints with the body containing a JSON Guestbook Response will save the response and +`?signed=true`: return a signed URL to download the file(s) or +`?signed=false` or missing: Write the guestbook responses and download the file(s) + +`/api/access/datafile/{fileId:.+}` +`/api/access/datafiles/{fileIds}` +`/api/access/dataset/{id}` +`/api/access/dataset/{id}/versions/{versionId}` + +A post to these endpoints with the body containing a JSON Guestbook Response will save the response before continuing the download. +No signed URL option exists. +`/api/access/datafiles` +`/api/access/datafile/bundle/{fileId}` POST returns BundleDownloadInstance after processing guestbook responses from body. ## New CRUD Endpoints for Guestbook: Create a Guestbook: POST `/api/guestbooks/{dataverseIdentifier}` diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 573e2ff79f7..ce7b686476b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -34,10 +34,7 @@ import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import jakarta.ejb.EJB; import jakarta.inject.Inject; -import jakarta.json.Json; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; +import jakarta.json.*; import jakarta.persistence.TypedQuery; import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.*; @@ -55,11 +52,10 @@ import java.io.*; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; @@ -143,13 +139,18 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr GuestbookResponse gbr = null; DataFile df = findDataFileOrDieWrapper(fileId); + User requestor = getRequestor(crc); // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); + + if (checkGuestbookRequiredResponse(requestor, df)) { + throw new BadRequestException(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + } if (gbrecs != true && df.isReleased()){ // Write Guestbook record if not done previously and file is released - gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, getRequestor(crc)); + gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, requestor); guestbookResponseService.save(gbr); MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); mdcLogService.logEntry(entry); @@ -191,7 +192,22 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr return downloadInstance; } - + + @POST + @AuthRequired + @Path("datafile/bundle/{fileId}") + @Produces({"application/zip"}) + public BundleDownloadInstance datafileBundleWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, @QueryParam("gbrecs") boolean gbrecs, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + Response res = processDatafileWithGuestbookResponse(crc, fileId, uriInfo, gbrecs, false, jsonBody); + if (res != null) { + throw new WebApplicationException(res); // must be an error since signed url is not an option + } else { + // return the download instance + return datafileBundle(crc, fileId, fileMetadataId, gbrecs, uriInfo, headers, response); + } + } + //Added a wrapper method since the original method throws a wrapped response //the access methods return files instead of responses so we convert to a WebApplicationException @@ -215,21 +231,8 @@ private DataFile findDataFileOrDieWrapper(String fileId){ @Path("datafile/{fileId:.+}") @Produces({"application/xml","*/*"}) public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - - // check first if there's a trailing slash, and chop it: - while (fileId.lastIndexOf('/') == fileId.length() - 1) { - fileId = fileId.substring(0, fileId.length() - 1); - } - - if (fileId.indexOf('/') > -1) { - // This is for embedding folder names into the Access API URLs; - // something like /api/access/datafile/folder/subfolder/1234 - // instead of the normal /api/access/datafile/1234 notation. - // this is supported only for recreating folders during recursive downloads - - // i.e. they are embedded into the URL for the remote client like wget, - // but can be safely ignored here. - fileId = fileId.substring(fileId.lastIndexOf('/') + 1); - } + + fileId = normalizeFileId(fileId); DataFile df = findDataFileOrDieWrapper(fileId); GuestbookResponse gbr = null; @@ -369,63 +372,103 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI @AuthRequired @Path("datafile/{fileId:.+}") @Produces({"application/json"}) - public Response datafileWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, + public Response datafileWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, + @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) { + fileId = normalizeFileId(fileId); + Response res = processDatafileWithGuestbookResponse(crc, fileId, uriInfo, gbrecs, signed, jsonBody); + if (res != null) { + return res; // could be an error or a signedUrl in the response + } else { + // initiate the download now + return datafile(crc, fileId, gbrecs, uriInfo, headers, response); + } + } + + private String normalizeFileId(String fileId) { + String fId = fileId; // check first if there's a trailing slash, and chop it: - while (fileId.lastIndexOf('/') == fileId.length() - 1) { - fileId = fileId.substring(0, fileId.length() - 1); + while (fId.lastIndexOf('/') == fId.length() - 1) { + fId = fId.substring(0, fId.length() - 1); } - if (fileId.indexOf('/') > -1) { + if (fId.indexOf('/') > -1) { // This is for embedding folder names into the Access API URLs; // something like /api/access/datafile/folder/subfolder/1234 // instead of the normal /api/access/datafile/1234 notation. // this is supported only for recreating folders during recursive downloads - // i.e. they are embedded into the URL for the remote client like wget, // but can be safely ignored here. - fileId = fileId.substring(fileId.lastIndexOf('/') + 1); + fId = fId.substring(fId.lastIndexOf('/') + 1); } + return fId; + } + private Response processDatafileWithGuestbookResponse(ContainerRequestContext crc, String fileIds, UriInfo uriInfo, boolean gbrecs, boolean signed, String jsonBody) { + String fileIdParams[] = getFileIdsCSV(fileIds); + AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); + Map datafilesMap = new HashMap<>(); - DataFile df = findDataFileOrDieWrapper(fileId); - GuestbookResponse gbr = null; + // Get and validate all the DataFiles first + if (fileIdParams != null && fileIdParams.length > 0) { + for (int i = 0; i < fileIdParams.length; i++) { + DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); - if (df.isHarvested()) { - String errorMessage = "Datafile " + fileId + " is a harvested file that cannot be accessed in this Dataverse"; - throw new NotFoundException(errorMessage); - // (nobody should ever be using this API on a harvested DataFile)! - } + if (df.isHarvested()) { + String errorMessage = "Datafile " + df.getId() + " is a harvested file that cannot be accessed in this Dataverse"; + throw new NotFoundException(errorMessage); + // (nobody should ever be using this API on a harvested DataFile)! + } - // This will throw a ForbiddenException if access isn't authorized: - checkAuthorization(crc, df); + // This will throw a ForbiddenException if access isn't authorized: + checkAuthorization(crc, df); - AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); - try { - if (checkGuestbookRequiredResponse(crc, df)) { - gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, jsonBody, user); - if (gbr != null) { - engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); - } else { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); - } - } else if (gbrecs != true && df.isReleased()) { - // Write Guestbook record if not done previously and file is released - gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, user); + datafilesMap.put(df.getId(), df); } - } catch (JsonParseException | CommandException ex) { - List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } - String baseUrl = uriInfo.getAbsolutePath().toString() + "?gbrecs=true"; - String key = ""; - ApiToken apiToken = authSvc.findApiTokenByUser(user); - if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { - key = apiToken.getTokenString(); + // Handle Guestbook Responses + for (DataFile df : datafilesMap.values()) { + try { + if (checkGuestbookRequiredResponse(user, df)) { + GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, jsonBody, user); + if (gbr != null) { + engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); + } else { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + } + } else if (gbrecs != true && df.isReleased()) { + // Write Guestbook record if not done previously and file is released + guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, user); + } + } catch (JsonParseException | CommandException ex) { + List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); + } + } + if (signed) { + return returnSignedUrl(datafilesMap, uriInfo, user); + } else { + return null; } - String signedUrl = UrlSignerUtil.signUrl(baseUrl, 10, user.getUserIdentifier(), "GET", key); + } - return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); + private Response returnSignedUrl(Map datafilesMap, UriInfo uriInfo, User user) { + AuthenticatedUser requestor = (AuthenticatedUser) user; + // Create the signed URL + if (!datafilesMap.isEmpty()) { + String baseUrlEncoded = uriInfo.getAbsolutePath() + "?gbrecs=true"; + String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); + String key = ""; + ApiToken apiToken = authSvc.findApiTokenByUser(requestor); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { + key = apiToken.getTokenString(); + } + String signedUrl = UrlSignerUtil.signUrl(baseUrl, 10, requestor.getUserIdentifier(), "GET", key); + return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); + } else { + return notFound("no file ids were given"); + } } /* @@ -651,7 +694,7 @@ public DownloadInstance downloadAuxiliaryFile(@Context ContainerRequestContext c /* * API method for downloading zipped bundles of multiple files. Uses POST to avoid long lists of file IDs that can make the URL longer than what's supported by browsers/servers */ - + // TODO: Rather than only supporting looking up files by their database IDs, // consider supporting persistent identifiers. @POST @@ -659,10 +702,15 @@ public DownloadInstance downloadAuxiliaryFile(@Context ContainerRequestContext c @Path("datafiles") @Consumes("text/plain") @Produces({ "application/zip" }) - public Response postDownloadDatafiles(@Context ContainerRequestContext crc, String fileIds, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - + public Response postDownloadDatafiles(@Context ContainerRequestContext crc, String body, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); + Response res = processDatafileWithGuestbookResponse(crc, body, uriInfo, gbrecs, false, body); + if (res != null) { + return res; // must be an error since signed url is not an option + } else { + // initiate the download now + return downloadDatafiles(crc, body, gbrecs, uriInfo, headers, response, null); + } } @GET @@ -709,37 +757,51 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat return wr.getResponse(); } } - - @GET + @POST @AuthRequired - @Path("dataset/{id}/versions/{versionId}") + @Path("dataset/{id}") @Produces({"application/zip"}) - public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @PathParam("versionId") String versionId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + public Response downloadAllFromLatestWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) throws WebApplicationException { try { - DataverseRequest req = createDataverseRequest(getRequestUser(crc)); - final Dataset ds = execCommand(new GetDatasetCommand(req, findDatasetOrDie(datasetIdOrPersistentId))); - DatasetVersion dsv = execCommand(handleVersion(versionId, new Datasets.DsVersionHandler>() { - - @Override - public Command handleLatest() { - return new GetLatestAccessibleDatasetVersionCommand(req, ds); - } - - @Override - public Command handleDraft() { - return new GetDraftDatasetVersionCommand(req, ds); + User user = getRequestUser(crc); + DataverseRequest req = createDataverseRequest(user); + final Dataset retrieved = findDatasetOrDie(datasetIdOrPersistentId); + String fileIds = ""; + String version = null; + // If user can view the draft version download those files and don't count them + if (!(user instanceof GuestUser)) { + final DatasetVersion draft = versionService.getDatasetVersionById(retrieved.getId(), DatasetVersion.VersionState.DRAFT.toString()); + if (draft != null && permissionService.requestOn(req, retrieved).has(Permission.ViewUnpublishedDataset)) { + fileIds = getFileIdsAsCommaSeparated(draft.getFileMetadatas()); + gbrecs = true; + version = "draft"; } + } + if (version == null) { + final DatasetVersion latest = versionService.getLatestReleasedVersionFast(retrieved.getId()); + fileIds = getFileIdsAsCommaSeparated(latest.getFileMetadatas()); + version = latest.getFriendlyVersionNumber(); + } + Response res = processDatafileWithGuestbookResponse(crc, fileIds, uriInfo, gbrecs, signed, jsonBody); - @Override - public Command handleSpecific(long major, long minor) { - return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor); - } + if (res != null) { + return res; // could be an error or a signedUrl in the response + } else { + // initiate the download now + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, version); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } - @Override - public Command handleLatestPublished() { - return new GetLatestPublishedDatasetVersionCommand(req, ds); - } - })); + @GET + @AuthRequired + @Path("dataset/{id}/versions/{versionId}") + @Produces({"application/zip"}) + public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @PathParam("versionId") String versionId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + try { + DatasetVersion dsv = getDatasetVersionFromVersion(crc, datasetIdOrPersistentId, versionId); if (dsv == null) { // (A "Not Found" would be more appropriate here, I believe, than a "Bad Request". // But we've been using the latter for a while, and it's a popular API... @@ -760,6 +822,55 @@ public Command handleLatestPublished() { } } + @POST + @AuthRequired + @Path("dataset/{id}/versions/{versionId}") + @Produces({"application/zip"}) + public Response downloadAllFromVersionWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @PathParam("versionId") String versionId, + @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @QueryParam("signed") Boolean signed, String jsonBody, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + try { + DatasetVersion dsv = getDatasetVersionFromVersion(crc, datasetIdOrPersistentId, versionId); + String fileIds = getFileIdsAsCommaSeparated(dsv.getFileMetadatas()); + Response res = processDatafileWithGuestbookResponse(crc, fileIds, uriInfo, gbrecs, signed, jsonBody); + if (res != null) { + return res; // could be an error or a signedUrl in the response + } else { + // initiate the download now + return downloadAllFromVersion(crc, datasetIdOrPersistentId, versionId, gbrecs, apiTokenParam, false, uriInfo, headers, response); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + + private DatasetVersion getDatasetVersionFromVersion(ContainerRequestContext crc, String datasetIdOrPersistentId, String versionId) throws WrappedResponse { + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + final Dataset ds = execCommand(new GetDatasetCommand(req, findDatasetOrDie(datasetIdOrPersistentId))); + return execCommand(handleVersion(versionId, new Datasets.DsVersionHandler<>() { + + @Override + public Command handleLatest() { + return new GetLatestAccessibleDatasetVersionCommand(req, ds); + } + + @Override + public Command handleDraft() { + return new GetDraftDatasetVersionCommand(req, ds); + } + + @Override + public Command handleSpecific(long major, long minor) { + return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor); + } + + @Override + public Command handleLatestPublished() { + return new GetLatestPublishedDatasetVersionCommand(req, ds); + } + })); + } + private static String getFileIdsAsCommaSeparated(List fileMetadatas) { List ids = new ArrayList<>(); for (FileMetadata fileMetadata : fileMetadatas) { @@ -794,192 +905,234 @@ private String generateMultiFileBundleName(Dataset dataset, String versionTag) { @AuthRequired @Path("datafiles/{fileIds}") @Produces({"application/zip"}) - public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); } - private Response downloadDatafiles(ContainerRequestContext crc, String rawFileIds, boolean donotwriteGBResponse, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response, String versionTag) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + @POST + @AuthRequired + @Path("datafiles/{fileIds}") + @Produces({"application/zip"}) + public Response datafilesWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) throws WebApplicationException { + + Response res = processDatafileWithGuestbookResponse(crc, fileIds, uriInfo, gbrecs, signed, jsonBody); + if (res != null) { + return res; // could be an error or a signedUrl in the response + } else { + // initiate the download now + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); + } + } + + private String[] getFileIdsCSV(String body) { + /* BODY has 3 variations coming from path parameter of GET or body of POST: + "1,2,3," + "fileIds=1,2,3" + {fileIds:[1,2,3], "guestbookResponse":{}} + */ + if (body.startsWith("fileIds=")) { + return body.substring(8).split(","); // Trim string "fileIds=" from the front + } else if (body.startsWith("{")) { // assume json + // get fileIds from json. example: {fileIds:[1,2,3], "guestbookResponse":{}} + JsonObject jsonObject = JsonUtil.getJsonObject(body); + if (jsonObject.containsKey("fileIds")) { + JsonArray ids = jsonObject.getJsonArray("fileIds"); + List idList = ids.getValuesAs(JsonNumber.class); + return idList.stream().map(JsonNumber::toString).toArray(String[]::new); + } else { + return new String[0]; + } + } else { + // default to expected list of ids "1,2,3" + return body.split(","); + } + } + + private Response downloadDatafiles(ContainerRequestContext crc, String body, boolean donotwriteGBResponse, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response, String versionTag) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { final long zipDownloadSizeLimit = systemConfig.getZipDownloadLimit(); - + logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes."); - - if (rawFileIds == null || rawFileIds.equals("")) { + + if (body == null || body.equals("")) { throw new BadRequestException(); } - - final String fileIds; - if(rawFileIds.startsWith("fileIds=")) { - fileIds = rawFileIds.substring(8); // String "fileIds=" from the front - } else { - fileIds=rawFileIds; - } + + String[] fileIdParams = getFileIdsCSV(body); + /* Note - fileIds coming from the POST ends in '\n' and a ',' has been added after the last file id number and before a * final '\n' - this stops the last item from being parsed in the fileIds.split(","); line below. */ - + String customZipServiceUrl = settingsService.getValueForKey(SettingsServiceBean.Key.CustomZipDownloadServiceUrl); - boolean useCustomZipService = customZipServiceUrl != null; + boolean useCustomZipService = customZipServiceUrl != null; User user = getRequestor(crc); - + Boolean getOrig = false; for (String key : uriInfo.getQueryParameters().keySet()) { String value = uriInfo.getQueryParameters().getFirst(key); - if("format".equals(key) && "original".equals(value)) { + if ("format".equals(key) && "original".equals(value)) { getOrig = true; } } - + + Map datafilesMap = new HashMap<>(); + + // Get DataFiles, check for multiple Datasets, and check for required guestbook response + Set datasetIds = new HashSet<>(); + for (int i = 0; i < fileIdParams.length; i++) { + if (!fileIdParams[i].isBlank()) { + DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); + datafilesMap.put(df.getId(), df); + datasetIds.add(df.getOwner() != null ? df.getOwner().getId() : 0L); + if (datasetIds.size() > 1) { + // All files must be from the same Dataset + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.multipleDatasets")); + } else if (checkGuestbookRequiredResponse(user, df)) { + try { + GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, body, user); + if (gbr != null) { + engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); + donotwriteGBResponse = true; + // Further down the actual download will also create a simple download response for every datafile listed based on the donotwriteGBResponse flag. + // Modifying donotwriteGBResponse will block that so we also need to log the MDC entry here + MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); + mdcLogService.logEntry(entry); + } else { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + } + } catch (JsonParseException | CommandException ex) { + List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); + } + } + } + } + if (useCustomZipService) { - URI redirect_uri = null; + URI redirect_uri = null; try { - redirect_uri = handleCustomZipDownload(user, customZipServiceUrl, fileIds, uriInfo, headers, donotwriteGBResponse, true); + redirect_uri = handleCustomZipDownload(user, customZipServiceUrl, fileIdParams, uriInfo, headers, donotwriteGBResponse, true); } catch (WebApplicationException wae) { throw wae; } - + Response redirect = Response.seeOther(redirect_uri).build(); logger.fine("Issuing redirect to the file location on S3."); throw new RedirectionException(redirect); } - - // Not using the "custom service" - API will zip the file, + + // Not using the "custom service" - API will zip the file, // and stream the output, in the "normal" manner: - - final boolean getOriginal = getOrig; //to use via anon inner class - + + // to use via anon inner class + final boolean getOriginal = getOrig; + final boolean skipGBResponse = donotwriteGBResponse; // Response may have been written prior and donotwriteGBResponse may have been modified. + StreamingOutput stream = new StreamingOutput() { @Override public void write(OutputStream os) throws IOException, WebApplicationException { - String fileIdParams[] = fileIds.split(","); - DataFileZipper zipper = null; + DataFileZipper zipper = null; String fileManifest = ""; long sizeTotal = 0L; - - if (fileIdParams != null && fileIdParams.length > 0) { - logger.fine(fileIdParams.length + " tokens;"); - for (int i = 0; i < fileIdParams.length; i++) { - logger.fine("token: " + fileIdParams[i]); - Long fileId = null; - try { - fileId = Long.parseLong(fileIdParams[i]); - } catch (NumberFormatException nfe) { - fileId = null; + + for (DataFile file : datafilesMap.values()) { + if (isAccessAuthorized(user, file)) { + logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance."); + //downloadInstance.addDataFile(file); + if (skipGBResponse != true && file.isReleased()) { + GuestbookResponse gbr = guestbookResponseService.initAPIGuestbookResponse(file.getOwner(), file, session, user); + guestbookResponseService.save(gbr); + MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, file); + mdcLogService.logEntry(entry); } - if (fileId != null) { - logger.fine("attempting to look up file id " + fileId); - DataFile file = dataFileService.find(fileId); - if (file != null) { - if (isAccessAuthorized(user, file)) { - - logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance."); - //downloadInstance.addDataFile(file); - if (donotwriteGBResponse != true && file.isReleased()){ - GuestbookResponse gbr = guestbookResponseService.initAPIGuestbookResponse(file.getOwner(), file, session, user); - guestbookResponseService.save(gbr); - MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, file); - mdcLogService.logEntry(entry); - } - - if (zipper == null) { - // This is the first file we can serve - so we now know that we are going to be able - // to produce some output. - zipper = new DataFileZipper(os); - zipper.setFileManifest(fileManifest); - String bundleName = generateMultiFileBundleName(file.getOwner(), versionTag); - response.setHeader("Content-disposition", "attachment; filename=\"" + bundleName + "\""); - response.setHeader("Content-Type", "application/zip; name=\"" + bundleName + "\""); - } - - long size = 0L; - // is the original format requested, and is this a tabular datafile, with a preserved original? - if (getOriginal - && file.isTabularData() - && !StringUtil.isEmpty(file.getDataTable().getOriginalFileFormat())) { - //This size check is probably fairly inefficient as we have to get all the AccessObjects - //We do this again inside the zipper. I don't think there is a better solution - //without doing a large deal of rewriting or architecture redo. - //The previous size checks for non-original download is still quick. - //-MAD 4.9.2 - // OK, here's the better solution: we now store the size of the original file in - // the database (in DataTable), so we get it for free. - // However, there may still be legacy datatables for which the size is not saved. - // so the "inefficient" code is kept, below, as a fallback solution. - // -- L.A., 4.10 - - if (file.getDataTable().getOriginalFileSize() != null) { - size = file.getDataTable().getOriginalFileSize(); - } else { - DataAccessRequest daReq = new DataAccessRequest(); - StorageIO storageIO = DataAccess.getStorageIO(file, daReq); - storageIO.open(); - size = storageIO.getAuxObjectSize(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); - - // save it permanently: - file.getDataTable().setOriginalFileSize(size); - fileService.saveDataTable(file.getDataTable()); - } - if (size == 0L){ - throw new IOException("Invalid file size or accessObject when checking limits of zip file"); - } - } else { - size = file.getFilesize(); - } - if (sizeTotal + size < zipDownloadSizeLimit) { - sizeTotal += zipper.addFileToZipStream(file, getOriginal); - } else { - String fileName = file.getFileMetadata().getLabel(); - String mimeType = file.getContentType(); - - zipper.addToManifest(fileName + " (" + mimeType + ") " + " skipped because the total size of the download bundle exceeded the limit of " + zipDownloadSizeLimit + " bytes.\r\n"); - } - } else { - boolean embargoed = FileUtil.isActivelyEmbargoed(file); - boolean retentionExpired = FileUtil.isRetentionExpired(file); - if (file.isRestricted() || embargoed || retentionExpired) { - if (zipper == null) { - fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") - + " AND CANNOT BE DOWNLOADED\r\n"; - } else { - zipper.addToManifest(file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") - + " AND CANNOT BE DOWNLOADED\r\n"); - } - } else { - fileId = null; - } - } - - } if (null == fileId) { - // As of now this errors out. - // This is bad because the user ends up with a broken zip and manifest - // This is good in that the zip ends early so the user does not wait for the results - String errorMessage = "Datafile " + fileId + ": no such object available"; - throw new NotFoundException(errorMessage); + + if (zipper == null) { + // This is the first file we can serve - so we now know that we are going to be able + // to produce some output. + zipper = new DataFileZipper(os); + zipper.setFileManifest(fileManifest); + String bundleName = generateMultiFileBundleName(file.getOwner(), versionTag); + response.setHeader("Content-disposition", "attachment; filename=\"" + bundleName + "\""); + response.setHeader("Content-Type", "application/zip; name=\"" + bundleName + "\""); + } + + long size = 0L; + // is the original format requested, and is this a tabular datafile, with a preserved original? + if (getOriginal + && file.isTabularData() + && !StringUtil.isEmpty(file.getDataTable().getOriginalFileFormat())) { + //This size check is probably fairly inefficient as we have to get all the AccessObjects + //We do this again inside the zipper. I don't think there is a better solution + //without doing a large deal of rewriting or architecture redo. + //The previous size checks for non-original download is still quick. + //-MAD 4.9.2 + // OK, here's the better solution: we now store the size of the original file in + // the database (in DataTable), so we get it for free. + // However, there may still be legacy datatables for which the size is not saved. + // so the "inefficient" code is kept, below, as a fallback solution. + // -- L.A., 4.10 + + if (file.getDataTable().getOriginalFileSize() != null) { + size = file.getDataTable().getOriginalFileSize(); + } else { + DataAccessRequest daReq = new DataAccessRequest(); + StorageIO storageIO = DataAccess.getStorageIO(file, daReq); + storageIO.open(); + size = storageIO.getAuxObjectSize(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); + + // save it permanently: + file.getDataTable().setOriginalFileSize(size); + fileService.saveDataTable(file.getDataTable()); + } + if (size == 0L) { + throw new IOException("Invalid file size or accessObject when checking limits of zip file"); + } + } else { + size = file.getFilesize(); + } + if (sizeTotal + size < zipDownloadSizeLimit) { + sizeTotal += zipper.addFileToZipStream(file, getOriginal); + } else { + String fileName = file.getFileMetadata().getLabel(); + String mimeType = file.getContentType(); + + zipper.addToManifest(fileName + " (" + mimeType + ") " + " skipped because the total size of the download bundle exceeded the limit of " + zipDownloadSizeLimit + " bytes.\r\n"); + } + } else { + boolean embargoed = FileUtil.isActivelyEmbargoed(file); + boolean retentionExpired = FileUtil.isRetentionExpired(file); + if (file.isRestricted() || embargoed || retentionExpired) { + if (zipper == null) { + fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS " + + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") + + " AND CANNOT BE DOWNLOADED\r\n"; + } else { + zipper.addToManifest(file.getFileMetadata().getLabel() + " IS " + + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") + + " AND CANNOT BE DOWNLOADED\r\n"); } } } - } else { - throw new BadRequestException(); } if (zipper == null) { - // If the DataFileZipper object is still NULL, it means that - // there were file ids supplied - but none of the corresponding - // files were accessible for this user. - // In which casew we don't bother generating any output, and + // If the DataFileZipper object is still NULL, it means that + // there were file ids supplied - but none of the corresponding + // files were accessible for this user. + // In which case we don't bother generating any output, and // just give them a 403: throw new ForbiddenException(); } - // This will add the generated File Manifest to the zipped output, + // This will add the generated File Manifest to the zipped output, // then flush and close the stream: zipper.finalizeZipStream(); - + //os.flush(); //os.close(); } @@ -1764,31 +1917,35 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @ return ok(jsonObjectBuilder); } - private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df) throws WebApplicationException { - // Check if guestbook response is required - if (df.isRestricted() && df.getOwner().hasEnabledGuestbook() && getRequestUser(crc) instanceof AuthenticatedUser) { - AuthenticatedUser user = (AuthenticatedUser)getRequestUser(crc); - List gbrList = guestbookResponseService.findByAuthenticatedUserId(user); - boolean responseFound = false; + private boolean checkGuestbookRequiredResponse(User user, DataFile df) throws WebApplicationException { + // Check if guestbook response is required and one does not already exist + boolean required = false; + if (df.isRestricted() && df.getOwner().hasEnabledGuestbook()) { + required = true; + // if we find an existing response for this user/datafile then it is not required to add another one + List gbrList = user instanceof AuthenticatedUser ? guestbookResponseService.findByAuthenticatedUserId((AuthenticatedUser)user) : null; if (gbrList != null) { - // find a matching response + // no need to check for nulls since if it's enabled it must exist + final Long guestbookId = df.getOwner().getGuestbook().getId(); + + // find a matching response for the datafile/guestbook combination + // this forces a new response if the guestbook changed for (GuestbookResponse r : gbrList) { - if (r.getDataFile().getId() == df.getId()) { - responseFound = true; + if (df.getId().equals(r.getDataFile().getId()) && guestbookId.equals(r.getGuestbook().getId())) { + required = false; break; } } } - return !responseFound; // if we find a response then it is not required to add another one } - return false; + return required; } private GuestbookResponse getGuestbookResponseFromBody(DataFile dataFile, String type, String jsonBody, User requestor) throws JsonParseException { Dataset ds = dataFile.getOwner(); GuestbookResponse guestbookResponse = null; - if (jsonBody != null && !jsonBody.isBlank()) { + if (jsonBody != null && jsonBody.startsWith("{")) { JsonObject guestbookResponseObj = JsonUtil.getJsonObject(jsonBody).getJsonObject("guestbookResponse"); guestbookResponse = guestbookResponseService.initAPIGuestbookResponse(ds, dataFile, null, requestor); guestbookResponse.setEventType(type); @@ -1935,12 +2092,11 @@ private boolean isAccessAuthorized(User requestUser, DataFile df) { return false; } - private URI handleCustomZipDownload(User user, String customZipServiceUrl, String fileIds, UriInfo uriInfo, HttpHeaders headers, boolean donotwriteGBResponse, boolean orig) throws WebApplicationException { + private URI handleCustomZipDownload(User user, String customZipServiceUrl, String[] fileIdParams, UriInfo uriInfo, HttpHeaders headers, boolean donotwriteGBResponse, boolean orig) throws WebApplicationException { String zipServiceKey = null; Timestamp timestamp = null; - - String fileIdParams[] = fileIds.split(","); + int validIdCount = 0; int validFileCount = 0; int downloadAuthCount = 0; diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f324af3762b..9282049e9fb 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2950,6 +2950,7 @@ access.api.exception.version.not.found=Could not find requested dataset version. access.api.exception.dataset.not.found=Could not find requested dataset. access.api.download.failure.guestbookResponseMissing=You may not download this file without the required Guestbook response. access.api.download.failure.guestbook.commandError=Problem trying download with guestbook response on {0} : {1} +access.api.download.failure.multipleDatasets=All files being downloaded must be from the same Dataset. #permission permission.AddDataverse.label=AddDataverse diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 54930a0953c..6a79444e504 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3875,82 +3875,98 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars msgt("testDownloadFileWithGuestbookResponse"); // Create superuser Response createUserResponse = UtilIT.createRandomUser(); - String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + String ownerApiToken = UtilIT.getApiTokenFromResponse(createUserResponse); String superusername = UtilIT.getUsernameFromResponse(createUserResponse); UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200); // Create Dataverse - String dataverseAlias = createDataverseGetAlias(apiToken); + String dataverseAlias = createDataverseGetAlias(ownerApiToken); // Create user with no permission createUserResponse = UtilIT.createRandomUser(); assertEquals(200, createUserResponse.getStatusCode()); - String apiTokenRando = UtilIT.getApiTokenFromResponse(createUserResponse); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); String username = UtilIT.getUsernameFromResponse(createUserResponse); // Create Dataset - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, ownerApiToken); createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); String persistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); - Response getDatasetMetadata = UtilIT.nativeGet(datasetId, apiToken); + Response getDatasetMetadata = UtilIT.nativeGet(datasetId, ownerApiToken); getDatasetMetadata.then().assertThat().statusCode(200); - Response getGuestbooksResponse = UtilIT.getGuestbooks(dataverseAlias, apiToken); + Response getGuestbooksResponse = UtilIT.getGuestbooks(dataverseAlias, ownerApiToken); getGuestbooksResponse.then().assertThat().statusCode(200); assertTrue(getGuestbooksResponse.getBody().jsonPath().getList("data").isEmpty()); // Create a Guestbook - Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, apiToken); + Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, ownerApiToken); // Get the list of Guestbooks - getGuestbooksResponse = UtilIT.getGuestbooks(dataverseAlias, apiToken); + getGuestbooksResponse = UtilIT.getGuestbooks(dataverseAlias, ownerApiToken); getGuestbooksResponse.then().assertThat().statusCode(200); assertEquals(1, getGuestbooksResponse.getBody().jsonPath().getList("data").size()); - // Upload file - String pathToFile1 = "src/main/webapp/resources/images/dataverseproject.png"; - JsonObjectBuilder json1 = Json.createObjectBuilder() - .add("description", "my description1") - .add("directoryLabel", "data/subdir1") - .add("categories", Json.createArrayBuilder().add("Data")); - Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile1, json1.build(), apiToken); + // Upload files + JsonObjectBuilder json1 = Json.createObjectBuilder().add("description", "my description1").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/dataverseproject.png", json1.build(), ownerApiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId1 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + JsonObjectBuilder json2 = Json.createObjectBuilder().add("description", "my description2").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/orcid_16x16.png", json1.build(), ownerApiToken); uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); - Integer fileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); - // Restrict file - Response restrictResponse = UtilIT.restrictFile(fileId.toString(), true, apiToken); + Integer fileId2 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + JsonObjectBuilder json3 = Json.createObjectBuilder().add("description", "my description3").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/cc0.png", json1.build(), ownerApiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId3 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + + // Restrict files + Response restrictResponse = UtilIT.restrictFile(fileId1.toString(), true, ownerApiToken); restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); + restrictResponse = UtilIT.restrictFile(fileId2.toString(), true, ownerApiToken); + restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); + restrictResponse = UtilIT.restrictFile(fileId3.toString(), true, ownerApiToken); + restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); + // Update Dataset to allow requests - Response allowAccessRequestsResponse = UtilIT.allowAccessRequests(datasetId.toString(), true, apiToken); + Response allowAccessRequestsResponse = UtilIT.allowAccessRequests(datasetId.toString(), true, ownerApiToken); assertEquals(200, allowAccessRequestsResponse.getStatusCode()); // Publish dataverse and dataset - Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, ownerApiToken); assertEquals(200, publishDataverse.getStatusCode()); - Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", ownerApiToken); assertEquals(200, publishDataset.getStatusCode()); // Request access - Response requestFileAccessResponse = UtilIT.requestFileAccess(fileId.toString(), apiTokenRando, null); - requestFileAccessResponse.prettyPrint(); + Response requestFileAccessResponse = UtilIT.requestFileAccess(fileId1.toString(), apiToken, null); + assertEquals(200, requestFileAccessResponse.getStatusCode()); + requestFileAccessResponse = UtilIT.requestFileAccess(fileId2.toString(), apiToken, null); + assertEquals(200, requestFileAccessResponse.getStatusCode()); + requestFileAccessResponse = UtilIT.requestFileAccess(fileId3.toString(), apiToken, null); assertEquals(200, requestFileAccessResponse.getStatusCode()); // Grant file access - Response grantFileAccessResponse = UtilIT.grantFileAccess(fileId.toString(), "@" + username, apiToken); - grantFileAccessResponse.prettyPrint(); + Response grantFileAccessResponse = UtilIT.grantFileAccess(fileId1.toString(), "@" + username, ownerApiToken); + assertEquals(200, grantFileAccessResponse.getStatusCode()); + grantFileAccessResponse = UtilIT.grantFileAccess(fileId2.toString(), "@" + username, ownerApiToken); + assertEquals(200, grantFileAccessResponse.getStatusCode()); + grantFileAccessResponse = UtilIT.grantFileAccess(fileId3.toString(), "@" + username, ownerApiToken); assertEquals(200, grantFileAccessResponse.getStatusCode()); String guestbookResponse = UtilIT.generateGuestbookResponse(guestbook); // Get Download Url attempt - Guestbook Response is required but not found - Response downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId, apiTokenRando, null); + Response downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken, null, false); downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) .statusCode(BAD_REQUEST.getStatusCode()); - // Get Download Url with guestbook response - downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId, apiTokenRando, guestbookResponse); + // Get Signed Download Url with guestbook response + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken, guestbookResponse, true); downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .statusCode(OK.getStatusCode()); @@ -3960,5 +3976,26 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars Response signedUrlResponse = get(signedUrl); signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); + + // Download multiple files - Guestbook Response is required but not found for file2 and file3 + downloadResponse = UtilIT.postDownloadDatafiles(fileId1 + "," + fileId2+ "," + fileId3, apiToken); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .body("status", equalTo(ApiConstants.STATUS_ERROR)) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) + .statusCode(BAD_REQUEST.getStatusCode()); + + // Download multiple files with guestbook response and fileIds in json + String jsonBody = "{\"fileIds\":[" + fileId1 + "," + fileId2+ "," + fileId3 +"], " + guestbookResponse.substring(1); + downloadResponse = UtilIT.postDownloadDatafiles(jsonBody, apiToken); + downloadResponse.prettyPrint(); + assertEquals(OK.getStatusCode(), downloadResponse.getStatusCode()); + + downloadResponse = UtilIT.downloadFilesUrlWithGuestbookResponse(new Integer[]{fileId1, fileId2, fileId3}, apiToken, guestbookResponse, true); + downloadResponse.prettyPrint(); + signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); + signedUrlResponse = get(signedUrl); + signedUrlResponse.prettyPrint(); + assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 72397d87c5f..5e43f2a72ae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1252,15 +1252,41 @@ static Response downloadFileOriginal(Integer fileId, String apiToken) { return given() .get("/api/access/datafile/" + fileId + "?format=original&key=" + apiToken); } - static Response getDownloadFileUrlWithGuestbookResponse(Integer fileId, String apiToken, String body) { + + static Response getDownloadFileUrlWithGuestbookResponse(Integer fileId, String apiToken, String body, boolean signed) { RequestSpecification requestSpecification = given(); requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); + String signedParam = signed ? "?signed=true" : ""; if (body != null) { requestSpecification.body(body); } - return requestSpecification.post("/api/access/datafile/" + fileId); + return requestSpecification.post("/api/access/datafile/" + fileId + signedParam); } - + + static Response downloadFilesUrlWithGuestbookResponse(Integer[] fileIds, String apiToken, String body, boolean signed) { + RequestSpecification requestSpecification = given(); + requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); + String signedParam = signed ? "?signed=true" : ""; + if (body != null) { + requestSpecification.body(body); + } + String getString = "/api/access/datafiles/"; + for (Integer fileId : fileIds) { + getString += fileId + ","; + } + return requestSpecification.post(getString + signedParam); + } + + static Response postDownloadDatafiles(String body, String apiToken) { + String getString = "/api/access/datafiles"; + RequestSpecification requestSpecification = given(); + requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); + if (body != null) { // body contains list of data file ids + requestSpecification.body(body); + } + return requestSpecification.post(getString); + } + static Response downloadFiles(Integer[] fileIds) { String getString = "/api/access/datafiles/"; for(Integer fileId : fileIds) { From e7bf66cbb5ea6efa6fea2bc00b27a47e0169980f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:27:23 -0500 Subject: [PATCH 17/52] fix accessIT test --- .../java/edu/harvard/iq/dataverse/api/Access.java | 12 ++++++++---- .../edu/harvard/iq/dataverse/api/AccessIT.java | 14 ++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index ce7b686476b..6ac56a65ae6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -905,7 +905,7 @@ private String generateMultiFileBundleName(Dataset dataset, String versionTag) { @AuthRequired @Path("datafiles/{fileIds}") @Produces({"application/zip"}) - public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); } @@ -978,18 +978,22 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo } Map datafilesMap = new HashMap<>(); + List authorizedDatafileIds = new ArrayList<>(); - // Get DataFiles, check for multiple Datasets, and check for required guestbook response + // Get DataFiles, check authorized access, check for multiple Datasets, and check for required guestbook response Set datasetIds = new HashSet<>(); for (int i = 0; i < fileIdParams.length; i++) { if (!fileIdParams[i].isBlank()) { DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); datafilesMap.put(df.getId(), df); datasetIds.add(df.getOwner() != null ? df.getOwner().getId() : 0L); + if (isAccessAuthorized(user, df)) { + authorizedDatafileIds.add(df.getId()); + } if (datasetIds.size() > 1) { // All files must be from the same Dataset return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.multipleDatasets")); - } else if (checkGuestbookRequiredResponse(user, df)) { + } else if (authorizedDatafileIds.contains(df.getId()) && checkGuestbookRequiredResponse(user, df)) { try { GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, body, user); if (gbr != null) { @@ -1041,7 +1045,7 @@ public void write(OutputStream os) throws IOException, long sizeTotal = 0L; for (DataFile file : datafilesMap.values()) { - if (isAccessAuthorized(user, file)) { + if (authorizedDatafileIds.contains(file.getId())) { logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance."); //downloadInstance.addDataFile(file); if (skipGBResponse != true && file.isReleased()) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index a2f5ff26eac..f4cb2cc6b12 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -159,6 +159,7 @@ public static void setUp() throws InterruptedException { tabFile4NameUnpublishedConvert = tabFile4NameUnpublished.substring(0, tabFile4NameUnpublished.indexOf(".dta")) + ".tab"; String tab4PathToFile = "scripts/search/data/tabular/" + tabFile4NameUnpublished; Response tab4AddResponse = UtilIT.uploadFileViaNative(datasetId.toString(), tab4PathToFile, apiToken); + tab4AddResponse.prettyPrint(); tabFile4IdUnpublished = JsonPath.from(tab4AddResponse.body().asString()).getInt("data.files[0].dataFile.id"); assertTrue(UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if Ingest Lock exceeds max duration " + tabFile2Name); @@ -412,18 +413,23 @@ public void testDownloadMultipleFiles_LoggedAndNot_Unpublished() throws IOExcept HashMap files2 = readZipResponse(authDownloadConvertedUnpublished.getBody().asInputStream()); assertEquals(4, files2.size()); //size +1 for manifest, we have access to unpublished + // Guest User can not access tabFile4IdUnpublished so only the first 2 files will be downloaded Response anonDownloadOriginalUnpublished = UtilIT.downloadFilesOriginal(new Integer[]{basicFileId,tabFile1Id,tabFile4IdUnpublished}); - assertEquals(404, anonDownloadOriginalUnpublished.getStatusCode()); + assertEquals(200, anonDownloadOriginalUnpublished.getStatusCode()); int origAnonSize = anonDownloadOriginalUnpublished.getBody().asByteArray().length; HashMap files3 = readZipResponse(anonDownloadOriginalUnpublished.getBody().asInputStream()); - assertEquals(0, files3.size()); //A size of 0 indicates the zip creation was interrupted. + // expect the zip to have 3 files: 2 downloaded files plus the manifest + assertEquals(3, files3.size()); + assertTrue(files3.containsKey("120745.dta")); assertTrue(origAnonSize < origAuthSize + margin); Response anonDownloadConvertedUnpublished = UtilIT.downloadFiles(new Integer[]{basicFileId,tabFile1Id,tabFile4IdUnpublished}); - assertEquals(404, anonDownloadConvertedUnpublished.getStatusCode()); + assertEquals(200, anonDownloadConvertedUnpublished.getStatusCode()); int convertAnonSize = anonDownloadConvertedUnpublished.getBody().asByteArray().length; HashMap files4 = readZipResponse(anonDownloadConvertedUnpublished.getBody().asInputStream()); - assertEquals(0, files4.size()); //A size of 0 indicates the zip creation was interrupted. + // expect the zip to have 3 files: 2 downloaded files plus the manifest + assertEquals(3, files4.size()); + assertTrue(files4.containsKey("120745.tab")); assertTrue(convertAnonSize < convertAuthSize + margin); } From 7c71e995f0dc9919c08039b1381d8d920c8dc931 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Feb 2026 13:52:50 -0500 Subject: [PATCH 18/52] fix to zipper manifest to add NOT Authorized files --- .../edu/harvard/iq/dataverse/api/Access.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 6ac56a65ae6..7239c19de38 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1110,16 +1110,16 @@ public void write(OutputStream os) throws IOException, } else { boolean embargoed = FileUtil.isActivelyEmbargoed(file); boolean retentionExpired = FileUtil.isRetentionExpired(file); - if (file.isRestricted() || embargoed || retentionExpired) { - if (zipper == null) { - fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") - + " AND CANNOT BE DOWNLOADED\r\n"; - } else { - zipper.addToManifest(file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") - + " AND CANNOT BE DOWNLOADED\r\n"); - } + String manifestEntry = file.getFileMetadata().getLabel() + " IS " + ( + embargoed ? "EMBARGOED" : + retentionExpired ? "RETENTIONEXPIRED" : + file.isRestricted() ? "RESTRICTED" : + "NOTAUTHORIZED") + + " AND CANNOT BE DOWNLOADED\r\n"; + if (zipper == null) { + fileManifest = fileManifest + manifestEntry; + } else { + zipper.addToManifest(manifestEntry); } } } From 3466d4e2a75945cf9a77def160aedd860ea8a314 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Feb 2026 14:56:40 -0500 Subject: [PATCH 19/52] update docs --- .../12001-api-support-termofuse-guestbook.md | 10 +++++----- doc/sphinx-guides/source/api/dataaccess.rst | 12 +++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md index ca0600eb6d0..d8738ebb114 100644 --- a/doc/release-notes/12001-api-support-termofuse-guestbook.md +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -1,9 +1,9 @@ ## Feature Request: API to support Download Terms of Use and Guestbook -## New Endpoints to download a file or files that required a Guestbook response: POST +## New Endpoints to download a file or files that required a Guestbook Response: POST A post to these endpoints with the body containing a JSON Guestbook Response will save the response and `?signed=true`: return a signed URL to download the file(s) or -`?signed=false` or missing: Write the guestbook responses and download the file(s) +`?signed=false` or missing: Write the Guestbook Responses and download the file(s) `/api/access/datafile/{fileId:.+}` `/api/access/datafiles/{fileIds}` @@ -13,7 +13,7 @@ A post to these endpoints with the body containing a JSON Guestbook Response wil A post to these endpoints with the body containing a JSON Guestbook Response will save the response before continuing the download. No signed URL option exists. `/api/access/datafiles` -`/api/access/datafile/bundle/{fileId}` POST returns BundleDownloadInstance after processing guestbook responses from body. +`/api/access/datafile/bundle/{fileId}` POST returns BundleDownloadInstance after processing Guestbook Responses from body. ## New CRUD Endpoints for Guestbook: Create a Guestbook: POST `/api/guestbooks/{dataverseIdentifier}` @@ -23,5 +23,5 @@ Enable/Disable a Guestbook: PUT `/api/guestbooks/{dataverseIdentifier}/{id}/enab Note: There is no Update or Delete at this time. You can disable a Guestbook and create a new one. ## For Guestbook At Request: -When JVM setting -Ddataverse.files.guestbook-at-request=true is used a request for access may require a Guestbook response. -PUT `/api/access/datafile/{id}/requestAccess` will now take a JSON Guestbook response in the body. +When JVM setting -Ddataverse.files.guestbook-at-request=true is used a request for access may require a Guestbook Response. +PUT `/api/access/datafile/{id}/requestAccess` will now take a JSON Guestbook Response in the body. diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 036b7920b8b..b0833e8e473 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,6 +32,8 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + A curl example using a DOI (no version): .. code-block:: bash @@ -59,6 +61,8 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + A curl example using a DOI (with version): .. code-block:: bash @@ -91,11 +95,11 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Restricted files that require a Guestbook response will require an additional step to supply the response. A POST to the same endpoint with the Guestbook Response in the body will return a signed url that can be used to download the file. +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. Example :: - POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -d '{"guestbookResponse": {"answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' + POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB&signed=true -d '{"guestbookResponse": {"answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' Parameters: ~~~~~~~~~~~ @@ -186,6 +190,8 @@ Returns the files listed, zipped. As of v6.7 the name of the zipped bundle will .. note:: If any of the datafiles have the ``DirectoryLabel`` attributes in the corresponding ``FileMetadata`` entries, these will be added as folders to the Zip archive, and the files will be placed in them accordingly. +.. note:: If Guestbook Responses are required they can be included in the body along with the file ids as JSON: ``{"fileIds" :[1,2,3], {"guestbookResponse": {"answers": []}}}``. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. + Parameters: ~~~~~~~~~~~ @@ -367,7 +373,7 @@ A curl example using an ``id``:: curl -H "X-Dataverse-key:$API_TOKEN" -X PUT http://$SERVER/api/access/datafile/{id}/requestAccess -.. note:: Some installations of Dataverse may require you to provide a Guestbook response when requesting access to certain restricted files. The response can be passed in the body of this call. See "Get a Guestbook for a Dataverse Collection" in the :doc:`native-api`. +.. note:: Some installations of Dataverse may require you to provide a Guestbook Response when requesting access to certain restricted files. The response can be passed in the body of this call. See "Get a Guestbook for a Dataverse Collection" in the :doc:`native-api`. Grant File Access: ~~~~~~~~~~~~~~~~~~ From ad22235ac32d84dd57af774a193ed58a385295e7 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 25 Feb 2026 09:10:51 -0500 Subject: [PATCH 20/52] adding signed param to GET endpoints --- .../12001-api-support-termofuse-guestbook.md | 2 + .../edu/harvard/iq/dataverse/api/Access.java | 96 +++++++++++++------ .../iq/dataverse/util/UrlSignerUtil.java | 38 +++++--- .../edu/harvard/iq/dataverse/api/FilesIT.java | 50 ++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++ .../iq/dataverse/util/UrlSignerUtilTest.java | 45 ++++++++- 6 files changed, 193 insertions(+), 44 deletions(-) diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md index d8738ebb114..6014270678f 100644 --- a/doc/release-notes/12001-api-support-termofuse-guestbook.md +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -10,6 +10,8 @@ A post to these endpoints with the body containing a JSON Guestbook Response wil `/api/access/dataset/{id}` `/api/access/dataset/{id}/versions/{versionId}` +The matching GET APIs will also take the `?signed=true` parameter to also return the signed url instead of downloading immediately. Note: Signed urls are only for Authenticated Users. Guest users will receive an error if requesting with signed=true' + A post to these endpoints with the body containing a JSON Guestbook Response will save the response before continuing the download. No signed URL option exists. `/api/access/datafiles` diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 7239c19de38..76194b21847 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -230,7 +230,13 @@ private DataFile findDataFileOrDieWrapper(String fileId){ @AuthRequired @Path("datafile/{fileId:.+}") @Produces({"application/xml","*/*"}) - public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + + if (signed) { + AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); + return returnSignedUrl(getDatafilesMap(crc, fileId), uriInfo, user, gbrecs); + } fileId = normalizeFileId(fileId); @@ -382,7 +388,7 @@ public Response datafileWithGuestbookResponse(@Context ContainerRequestContext c return res; // could be an error or a signedUrl in the response } else { // initiate the download now - return datafile(crc, fileId, gbrecs, uriInfo, headers, response); + return datafile(crc, fileId, gbrecs, false, uriInfo, headers, response); } } @@ -405,27 +411,9 @@ private String normalizeFileId(String fileId) { return fId; } private Response processDatafileWithGuestbookResponse(ContainerRequestContext crc, String fileIds, UriInfo uriInfo, boolean gbrecs, boolean signed, String jsonBody) { - String fileIdParams[] = getFileIdsCSV(fileIds); AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); - Map datafilesMap = new HashMap<>(); - // Get and validate all the DataFiles first - if (fileIdParams != null && fileIdParams.length > 0) { - for (int i = 0; i < fileIdParams.length; i++) { - DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); - - if (df.isHarvested()) { - String errorMessage = "Datafile " + df.getId() + " is a harvested file that cannot be accessed in this Dataverse"; - throw new NotFoundException(errorMessage); - // (nobody should ever be using this API on a harvested DataFile)! - } - - // This will throw a ForbiddenException if access isn't authorized: - checkAuthorization(crc, df); - - datafilesMap.put(df.getId(), df); - } - } + Map datafilesMap = getDatafilesMap(crc, fileIds); // Handle Guestbook Responses for (DataFile df : datafilesMap.values()) { @@ -440,6 +428,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr } else if (gbrecs != true && df.isReleased()) { // Write Guestbook record if not done previously and file is released guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, user); + gbrecs = true; // prevent it from being written again } } catch (JsonParseException | CommandException ex) { List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); @@ -447,17 +436,44 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr } } if (signed) { - return returnSignedUrl(datafilesMap, uriInfo, user); + return returnSignedUrl(datafilesMap, uriInfo, user, gbrecs); } else { return null; } } - private Response returnSignedUrl(Map datafilesMap, UriInfo uriInfo, User user) { + private Map getDatafilesMap(ContainerRequestContext crc, String fileIds) { + String fileIdParams[] = getFileIdsCSV(fileIds); + Map datafilesMap = new HashMap<>(); + // Get and validate all the DataFiles first + if (fileIdParams != null && fileIdParams.length > 0) { + for (int i = 0; i < fileIdParams.length; i++) { + DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); + + if (df.isHarvested()) { + String errorMessage = "Datafile " + df.getId() + " is a harvested file that cannot be accessed in this Dataverse"; + throw new NotFoundException(errorMessage); + // (nobody should ever be using this API on a harvested DataFile)! + } + + // This will throw a ForbiddenException if access isn't authorized: + checkAuthorization(crc, df); + + datafilesMap.put(df.getId(), df); + } + } + return datafilesMap; + } + + private Response returnSignedUrl(Map datafilesMap, UriInfo uriInfo, User user, boolean gbrecs) { AuthenticatedUser requestor = (AuthenticatedUser) user; // Create the signed URL if (!datafilesMap.isEmpty()) { - String baseUrlEncoded = uriInfo.getAbsolutePath() + "?gbrecs=true"; + UriBuilder builder = UriBuilder.fromUri(uriInfo.getRequestUri()); + builder.replaceQueryParam("gbrecs", String.valueOf(gbrecs)); + URI modifiedUri = builder.build(); + + String baseUrlEncoded = modifiedUri.toString();//uriInfo.getRequestUri().toString(); String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); String key = ""; ApiToken apiToken = authSvc.findApiTokenByUser(requestor); @@ -717,7 +733,8 @@ public Response postDownloadDatafiles(@Context ContainerRequestContext crc, Stri @AuthRequired @Path("dataset/{id}") @Produces({"application/zip"}) - public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { try { User user = getRequestUser(crc); DataverseRequest req = createDataverseRequest(user); @@ -731,7 +748,11 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat // We don't want downloads from Draft versions to be counted, // so we are setting the gbrecs (aka "do not write guestbook response") // variable accordingly: - return downloadDatafiles(crc, fileIds, true, uriInfo, headers, response, "draft"); + if (signed) { + return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, true); + } else { + return downloadDatafiles(crc, fileIds, true, uriInfo, headers, response, "draft"); + } } } @@ -752,7 +773,11 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat } String fileIds = getFileIdsAsCommaSeparated(latest.getFileMetadatas()); - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, latest.getFriendlyVersionNumber()); + if (signed) { + return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, gbrecs); + } else { + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, latest.getFriendlyVersionNumber()); + } } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -816,7 +841,12 @@ public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @Pa if (dsv.isDraft()) { gbrecs = true; } - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, dsv.getFriendlyVersionNumber().toLowerCase()); + if (signed) { + AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); + return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, gbrecs); + } else { + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, dsv.getFriendlyVersionNumber().toLowerCase()); + } } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -905,8 +935,14 @@ private String generateMultiFileBundleName(Dataset dataset, String versionTag) { @AuthRequired @Path("datafiles/{fileIds}") @Produces({"application/zip"}) - public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); + public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + if (signed) { + AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); + return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, gbrecs); + } else { + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); + } } @POST diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index 18ea3771301..8d81b9f57b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -1,20 +1,22 @@ package edu.harvard.iq.dataverse.util; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URLEncodedUtils; +import org.joda.time.LocalDateTime; + import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.joda.time.LocalDateTime; - /** * Simple class to sign/validate URLs. - * + * */ public class UrlSignerUtil { @@ -24,8 +26,11 @@ public class UrlSignerUtil { public static final String SIGNED_URL_METHOD="method"; public static final String SIGNED_URL_USER="user"; public static final String SIGNED_URL_UNTIL="until"; + public static final String SIGNED_URL_KEY="key"; // do not propagate the key since it's a credential + public static final String SIGNED_URL_SIGNED="signed"; // we need to remove this when returning a singed url to prevent a loop of signing + public static final List reservedParameters = List.of(SIGNED_URL_UNTIL, SIGNED_URL_USER, SIGNED_URL_METHOD, SIGNED_URL_TOKEN, SIGNED_URL_KEY, SIGNED_URL_SIGNED); /** - * + * * @param baseUrl - the URL to sign - cannot contain query params * "until","user", "method", or "token" * @param timeout - how many minutes to make the URL valid for (note - time skew @@ -39,12 +44,23 @@ public class UrlSignerUtil { * @return - the signed URL */ public static String signUrl(String baseUrl, Integer timeout, String user, String method, String key) { - StringBuilder signedUrlBuilder = new StringBuilder(baseUrl); - boolean firstParam = true; - if (baseUrl.contains("?")) { - firstParam = false; + // check for reserved parameter names ("until","user", "method", or "token") + String[] urlQP = baseUrl.split("\\?"); + if (urlQP.length > 1) { + try { + URIBuilder uriBuilder = new URIBuilder(baseUrl); + List params = uriBuilder.getQueryParams(); + params.removeIf(pair -> reservedParameters.contains(pair.getName())); + uriBuilder.setParameters(params); + baseUrl = uriBuilder.build().toString(); + } catch (URISyntaxException e) { + logger.severe("Invalid URL for signing: " + baseUrl + " " + e.getMessage()); + } } + boolean firstParam = !baseUrl.contains("?"); + StringBuilder signedUrlBuilder = new StringBuilder(baseUrl); + if (timeout != null) { LocalDateTime validTime = LocalDateTime.now(); validTime = validTime.plusMinutes(timeout); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 6a79444e504..604d3803666 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3998,4 +3998,54 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); } + + @Test + public void testDownloadFileWithSignedUrl() throws IOException, JsonParseException { + msgt("testDownloadFileWithSignedUrl"); + // Create superuser + Response createUserResponse = UtilIT.createRandomUser(); + String ownerApiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + String superusername = UtilIT.getUsernameFromResponse(createUserResponse); + UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200); + + // Create Dataverse + String dataverseAlias = createDataverseGetAlias(ownerApiToken); + + // Create user with no permission + createUserResponse = UtilIT.createRandomUser(); + assertEquals(200, createUserResponse.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + String username = UtilIT.getUsernameFromResponse(createUserResponse); + + // Create Dataset + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, ownerApiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String persistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + Response getDatasetMetadata = UtilIT.nativeGet(datasetId, ownerApiToken); + getDatasetMetadata.then().assertThat().statusCode(200); + + // Upload files + JsonObjectBuilder json1 = Json.createObjectBuilder().add("description", "my description1").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/dataverseproject.png", json1.build(), ownerApiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId1 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + JsonObjectBuilder json2 = Json.createObjectBuilder().add("description", "my description2").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/orcid_16x16.png", json1.build(), ownerApiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId2 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + JsonObjectBuilder json3 = Json.createObjectBuilder().add("description", "my description3").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/cc0.png", json1.build(), ownerApiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId3 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + + // Get Signed Download Url with guestbook response + // downloadFile(Integer fileId, String byteRange, String format, String imageThumb, String apiToken) + Response downloadResponse = UtilIT.downloadFile(fileId1, "&signed=true&format=original" , ownerApiToken); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + String signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); + + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 5e43f2a72ae..e4fb8e1053e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1231,6 +1231,12 @@ static Response downloadFile(Integer fileId, String byteRange, String format, St //.header(API_TOKEN_HTTP_HEADER, apiToken) return requestSpecification.get("/api/access/datafile/" + fileId + "?key=" + apiToken + optionalFormat + optionalImageThumb); } + + static Response downloadFile(Integer fileId, String queryParams, String apiToken) { + RequestSpecification requestSpecification = given(); + + return requestSpecification.get("/api/access/datafile/" + fileId + "?key=" + apiToken + queryParams); + } static Response downloadTabularFile(Integer fileId) { return given() diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java index 09739b67023..d92f8822e59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java @@ -1,12 +1,15 @@ package edu.harvard.iq.dataverse.util; -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertTrue; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.MultivaluedMap; +import org.junit.jupiter.api.Test; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class UrlSignerUtilTest { @@ -47,4 +50,40 @@ public void testSignAndValidate() { assertFalse(UrlSignerUtil.isValidUrl(signedUrl3, user1, get, key)); } + + @Test + public void testSignAndValidateWithParams() { + final String url1 = "http://localhost:8080/api/test1?p1=true&p2=test"; + final String url2 = "http://localhost:8080/api/test1?p1=true&p2=test&until=2999-01-01&user=Fred&method=POST&token=abracadabara&signed=true"; + final String url3 = "localhost:8080/api/test1?p1=true&p2&until=2099-01-01"; + final int longTimeout = 1000; + final String user1 = "Alice"; + final String key = "abracadabara open sesame"; + MultivaluedMap queryParameters = new MultivaluedHashMap<>(); + queryParameters.put("p1", List.of("true")); + queryParameters.put("p2", List.of("test")); + queryParameters.put("until", List.of("2099-01-01")); + + String signedUrl1 = UrlSignerUtil.signUrl(url1, longTimeout, user1, "GET", key); + assertTrue(signedUrl1.contains("test1?p1=true&p2=test")); + System.out.println(signedUrl1); + + String signedUrl2 = UrlSignerUtil.signUrl(url2, longTimeout, user1, "GET", key); + assertTrue(signedUrl2.contains("&until=")); // contains the until param but not the bogus one passed in + assertFalse(signedUrl2.contains("&until=2099-01-01")); + assertTrue(signedUrl2.contains("&user=Alice")); // contains the user param but not the bogus one passed in + assertFalse(signedUrl2.contains("&user=Fred")); + assertTrue(signedUrl2.contains("&method=GET")); // contains the method param but not the bogus one passed in + assertFalse(signedUrl2.contains("&method=POST")); + assertTrue(signedUrl2.contains("&token=")); // contains the signed token param but not the bogus one passed in + assertFalse(signedUrl2.contains("&token=abracadabara")); + assertFalse(signedUrl2.contains("&signed")); // make sure we don't propagate the "signed" param + System.out.println(signedUrl2); + + // This will log an error but will still return the signed url even if it's now a valid url + // All callers of this method don't handle errors being returned, and it's highly unlikely that the url would be bad + String signedUrl3 = UrlSignerUtil.signUrl(url3, longTimeout, user1, "GET", key); + System.out.println(signedUrl3); + assertTrue(signedUrl3.contains("&p2&")); // Show that this works with params that have no value + } } From cedd4c28d8afbea3a40b4cbc3d7b65cb4ef351dd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 25 Feb 2026 11:06:38 -0500 Subject: [PATCH 21/52] add overwrite to name, email, institution, and position in guestbook response JSON --- .../iq/dataverse/util/json/JsonParser.java | 13 +++++++--- .../edu/harvard/iq/dataverse/api/FilesIT.java | 25 +++++++++++++++++-- .../dataverse/util/json/JsonParserTest.java | 19 ++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 804a3c3cbee..a59b21b8ee1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -603,6 +603,11 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons if (obj == null || guestbookResponse == null || guestbookResponse.getGuestbook() == null || guestbookResponse.getGuestbook().getCustomQuestions() == null) { return null; } + // overwrite name, email, institution and position. + guestbookResponse.setName(obj.getString("name", guestbookResponse.getName())); + guestbookResponse.setEmail(obj.getString("email", guestbookResponse.getEmail())); + guestbookResponse.setInstitution(obj.getString("institution", guestbookResponse.getInstitution())); + guestbookResponse.setPosition(obj.getString("position", guestbookResponse.getPosition())); Map cqMap = new HashMap<>(); guestbookResponse.getGuestbook().getCustomQuestions().stream().forEach(cq -> cqMap.put(cq.getId(),cq)); JsonArray answers = obj.getJsonArray("answers"); @@ -637,14 +642,14 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons } guestbookResponse.setCustomQuestionResponses(customQuestionResponses); // verify each required question is in the response - List missingReponses = new ArrayList<>(); + List missingResponses = new ArrayList<>(); for (Map.Entry e : cqMap.entrySet()) { if (e.getValue().isRequired()) { - missingReponses.add(e.getValue().getQuestionString()); + missingResponses.add(e.getValue().getQuestionString()); } } - if (!missingReponses.isEmpty()) { - String missing = String.join(",", missingReponses); + if (!missingResponses.isEmpty()) { + String missing = String.join(",", missingResponses); throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseMissingRequired", List.of(missing))); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 604d3803666..4305981a28d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3888,6 +3888,14 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); String username = UtilIT.getUsernameFromResponse(createUserResponse); + // Create second user with no permission + createUserResponse = UtilIT.createRandomUser(); + createUserResponse.prettyPrint(); + assertEquals(200, createUserResponse.getStatusCode()); + String apiToken2 = UtilIT.getApiTokenFromResponse(createUserResponse); + String username2 = UtilIT.getUsernameFromResponse(createUserResponse); + String user2Email = JsonPath.from(createUserResponse.body().asString()).getString("data.authenticatedUser.email"); + // Create Dataset Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, ownerApiToken); createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); @@ -3997,6 +4005,21 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars signedUrlResponse = get(signedUrl); signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); + + // TEST Overwrite name, email, institution and position in guestbook Response. Using user2 + requestFileAccessResponse = UtilIT.requestFileAccess(fileId1.toString(), apiToken2, null); + assertEquals(200, requestFileAccessResponse.getStatusCode()); + grantFileAccessResponse = UtilIT.grantFileAccess(fileId1.toString(), "@" + username2, ownerApiToken); + assertEquals(200, grantFileAccessResponse.getStatusCode()); + // Modify guestbookResponse excluding email to show that the email remains unchanged + guestbookResponse = guestbookResponse.replace("\"guestbookResponse\": {", + "\"guestbookResponse\": { \"name\":\"My Name\", \"position\":\"My Position\", \"institution\":\"My Institution\","); + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken2, guestbookResponse, true); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + Response guestbookResponses = UtilIT.getGuestbookResponses(dataverseAlias, guestbook.getId(), ownerApiToken); + assertTrue(guestbookResponses.prettyPrint().contains("My Name," + user2Email + ",My Institution,My Position")); } @Test @@ -4045,7 +4068,5 @@ public void testDownloadFileWithSignedUrl() throws IOException, JsonParseExcepti downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .statusCode(OK.getStatusCode()); - String signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); - } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index 73451aeeb71..e062f1a3d57 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -811,6 +811,10 @@ public void testGuestbookResponse() throws JsonParseException { final String guestbookResponseJson = """ { + "name": "My Name", + "email": "myemail@example.com", + "institution": "Harvard", + "position": "Upright", "answers": [ { "id": 1, @@ -872,5 +876,20 @@ public void testGuestbookResponse() throws JsonParseException { System.out.println(e.getMessage()); assertTrue(e.getMessage().contains("ID 4 not found")); } + + // Test overwrite name, email, institution and position. + try { + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + assertEquals("My Name", gbr.getName()); + // Removing name from the JSON defaults it to the original value in guestbook response + gbr.setName("My Original Name"); + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("\"name\": \"My Name\",", "")); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + assertEquals("My Original Name", gbr.getName()); + } catch (JsonParseException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("ID 4 not found")); + } } } From dc9f8bc2d576cdb4eabf3e25d0cbc1eab3b0abea Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 25 Feb 2026 11:15:45 -0500 Subject: [PATCH 22/52] fix doc --- doc/release-notes/12001-api-support-termofuse-guestbook.md | 2 +- doc/sphinx-guides/source/api/dataaccess.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md index 6014270678f..daa3ddc63a2 100644 --- a/doc/release-notes/12001-api-support-termofuse-guestbook.md +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -10,7 +10,7 @@ A post to these endpoints with the body containing a JSON Guestbook Response wil `/api/access/dataset/{id}` `/api/access/dataset/{id}/versions/{versionId}` -The matching GET APIs will also take the `?signed=true` parameter to also return the signed url instead of downloading immediately. Note: Signed urls are only for Authenticated Users. Guest users will receive an error if requesting with signed=true' +The matching GET APIs will also take the `?signed=true` parameter to also return the signed url instead of downloading immediately. Note: Signed urls are only for Authenticated Users. Guest users will receive an error if requesting with signed=true A post to these endpoints with the body containing a JSON Guestbook Response will save the response before continuing the download. No signed URL option exists. diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 391390cdc55..315178ec6a9 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -99,7 +99,7 @@ Basic access URI: Example :: - POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB&signed=true -d '{"guestbookResponse": {"answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' + POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB&signed=true -d '{"guestbookResponse": {"name": "My Name", "email": "myemail@example.com", "institution": "Harvard","position": "Upright", "answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' Parameters: ~~~~~~~~~~~ From d6aa29a3099342898c32bc8dbf7d910c47ff125f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 25 Feb 2026 11:48:11 -0500 Subject: [PATCH 23/52] fix doc --- doc/sphinx-guides/source/api/dataaccess.rst | 4 ++-- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 315178ec6a9..e52f2c4fbab 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -95,11 +95,11 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Example :: - POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB&signed=true -d '{"guestbookResponse": {"name": "My Name", "email": "myemail@example.com", "institution": "Harvard","position": "Upright", "answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' + POST http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB&signed=true -d '{"guestbookResponse": {"name": "My Name", "email": "myemail@example.com", "institution": "Harvard","position": "Staff", "answers": [{"id": 123,"value": "Good"},{"id": 124,"value": ["Multi","Line"]},{"id": 125,"value": "Yellow"}]}}' Parameters: ~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 76194b21847..9993a6dd159 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -435,11 +435,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } } - if (signed) { - return returnSignedUrl(datafilesMap, uriInfo, user, gbrecs); - } else { - return null; - } + return signed ? returnSignedUrl(datafilesMap, uriInfo, user, gbrecs) : null; } private Map getDatafilesMap(ContainerRequestContext crc, String fileIds) { @@ -473,7 +469,7 @@ private Response returnSignedUrl(Map datafilesMap, UriInfo uriIn builder.replaceQueryParam("gbrecs", String.valueOf(gbrecs)); URI modifiedUri = builder.build(); - String baseUrlEncoded = modifiedUri.toString();//uriInfo.getRequestUri().toString(); + String baseUrlEncoded = modifiedUri.toString(); String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); String key = ""; ApiToken apiToken = authSvc.findApiTokenByUser(requestor); From 72fce965d638399105e5aacc777297af705f35a0 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:35:04 -0500 Subject: [PATCH 24/52] add email validation --- .../harvard/iq/dataverse/util/json/JsonParser.java | 12 ++++++++++-- .../iq/dataverse/util/json/JsonParserTest.java | 11 ++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index a59b21b8ee1..d271e49e09e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -19,6 +19,7 @@ import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; import jakarta.json.*; @@ -603,9 +604,16 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons if (obj == null || guestbookResponse == null || guestbookResponse.getGuestbook() == null || guestbookResponse.getGuestbook().getCustomQuestions() == null) { return null; } - // overwrite name, email, institution and position. + // overwrite name, email(if valid), institution and position. guestbookResponse.setName(obj.getString("name", guestbookResponse.getName())); - guestbookResponse.setEmail(obj.getString("email", guestbookResponse.getEmail())); + String email = obj.getString("email", null); + if (email != null) { + if (EMailValidator.isEmailValid(email)) { + guestbookResponse.setEmail(email); + } else { + logger.warning("Ignoring invalid email address in Guestbook response: " + email); + } + } guestbookResponse.setInstitution(obj.getString("institution", guestbookResponse.getInstitution())); guestbookResponse.setPosition(obj.getString("position", guestbookResponse.getPosition())); Map cqMap = new HashMap<>(); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index e062f1a3d57..e1bfbefe8d5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -812,7 +812,7 @@ public void testGuestbookResponse() throws JsonParseException { final String guestbookResponseJson = """ { "name": "My Name", - "email": "myemail@example.com", + "email": "my.email@example.com", "institution": "Harvard", "position": "Upright", "answers": [ @@ -887,6 +887,15 @@ public void testGuestbookResponse() throws JsonParseException { jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("\"name\": \"My Name\",", "")); gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); assertEquals("My Original Name", gbr.getName()); + // test invalid email (does not change original) + gbr.setEmail("original@example.com"); + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("my.email@example.com", "badEmail.com")); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + assertEquals("original@example.com", gbr.getEmail()); + // test valid email (overwrite email) + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("my.email@example.com", "new@example.com")); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + assertEquals("new@example.com", gbr.getEmail()); } catch (JsonParseException e) { System.out.println(e.getMessage()); assertTrue(e.getMessage().contains("ID 4 not found")); From cf2937b7f1533e182e2c8b1daf64bb5bf11f1267 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:39:50 -0500 Subject: [PATCH 25/52] add email validation --- src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index d271e49e09e..2cba4faceb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -608,6 +608,7 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons guestbookResponse.setName(obj.getString("name", guestbookResponse.getName())); String email = obj.getString("email", null); if (email != null) { + email = email.trim(); if (EMailValidator.isEmailValid(email)) { guestbookResponse.setEmail(email); } else { From 35c2f752a9a9326d29bfc778e3cdb80dd2c549c1 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 4 Mar 2026 14:59:24 -0500 Subject: [PATCH 26/52] allow guest to download with signedUrl --- .../edu/harvard/iq/dataverse/api/Access.java | 218 +++++++----------- .../api/auth/SignedUrlAuthMechanism.java | 19 +- .../iq/dataverse/util/json/JsonParser.java | 19 +- src/main/java/propertyFiles/Bundle.properties | 2 +- .../edu/harvard/iq/dataverse/api/FilesIT.java | 44 +++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 16 +- 6 files changed, 164 insertions(+), 154 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 9993a6dd159..e759b95e2dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -27,6 +27,7 @@ import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.*; import edu.harvard.iq.dataverse.util.json.JsonParseException; @@ -133,24 +134,22 @@ public class Access extends AbstractApiBean { @AuthRequired @Path("datafile/bundle/{fileId}") @Produces({"application/zip"}) - public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId,@QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - + public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, + @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - GuestbookResponse gbr = null; - DataFile df = findDataFileOrDieWrapper(fileId); - User requestor = getRequestor(crc); // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); - if (checkGuestbookRequiredResponse(requestor, df)) { + if (checkGuestbookRequiredResponse(crc, df)) { throw new BadRequestException(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } - if (gbrecs != true && df.isReleased()){ + if (gbrecs != true && df.isReleased()) { // Write Guestbook record if not done previously and file is released - gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, requestor); + User requestor = getRequestor(crc); + GuestbookResponse gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, requestor); guestbookResponseService.save(gbr); MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); mdcLogService.logEntry(entry); @@ -199,9 +198,10 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr @Produces({"application/zip"}) public BundleDownloadInstance datafileBundleWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - Response res = processDatafileWithGuestbookResponse(crc, fileId, uriInfo, gbrecs, false, jsonBody); + Response res = processDatafileWithGuestbookResponse(crc, headers, fileId, uriInfo, gbrecs, jsonBody); if (res != null) { - throw new WebApplicationException(res); // must be an error since signed url is not an option + throw new WebApplicationException(res); + // TODO: There is no get for this so we shouldn't return a signed url } else { // return the download instance return datafileBundle(crc, fileId, fileMetadataId, gbrecs, uriInfo, headers, response); @@ -230,14 +230,9 @@ private DataFile findDataFileOrDieWrapper(String fileId){ @AuthRequired @Path("datafile/{fileId:.+}") @Produces({"application/xml","*/*"}) - public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - if (signed) { - AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); - return returnSignedUrl(getDatafilesMap(crc, fileId), uriInfo, user, gbrecs); - } - fileId = normalizeFileId(fileId); DataFile df = findDataFileOrDieWrapper(fileId); @@ -251,6 +246,9 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); + if (checkGuestbookRequiredResponse(crc, df)) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + } if (gbrecs != true && df.isReleased()){ // Write Guestbook record if not done previously and file is released @@ -369,7 +367,7 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI * Provide some browser-friendly headers: (?) */ if (headers.getRequestHeaders().containsKey("Range")) { - return Response.status(Response.Status.PARTIAL_CONTENT).entity(downloadInstance).build(); + return Response.status(PARTIAL_CONTENT).entity(downloadInstance).build(); } return Response.ok(downloadInstance).build(); } @@ -379,17 +377,10 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI @Path("datafile/{fileId:.+}") @Produces({"application/json"}) public Response datafileWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, - @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, - @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) { + @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) { fileId = normalizeFileId(fileId); - Response res = processDatafileWithGuestbookResponse(crc, fileId, uriInfo, gbrecs, signed, jsonBody); - if (res != null) { - return res; // could be an error or a signedUrl in the response - } else { - // initiate the download now - return datafile(crc, fileId, gbrecs, false, uriInfo, headers, response); - } + return processDatafileWithGuestbookResponse(crc, headers, fileId, uriInfo, gbrecs, jsonBody); } private String normalizeFileId(String fileId) { @@ -410,32 +401,45 @@ private String normalizeFileId(String fileId) { } return fId; } - private Response processDatafileWithGuestbookResponse(ContainerRequestContext crc, String fileIds, UriInfo uriInfo, boolean gbrecs, boolean signed, String jsonBody) { - AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); + + // Process the guestbook response from JSON and return a signedUrl to the matching GET call + private Response processDatafileWithGuestbookResponse(ContainerRequestContext crc, HttpHeaders headers, String fileIds, UriInfo uriInfo, boolean gbrecs, String jsonBody) { + + User user = getRequestUser(crc); + // Get and validate all the DataFiles first Map datafilesMap = getDatafilesMap(crc, fileIds); // Handle Guestbook Responses - for (DataFile df : datafilesMap.values()) { - try { - if (checkGuestbookRequiredResponse(user, df)) { - GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, jsonBody, user); + String displayName = ""; + try { + // since all files must be in the same Dataset we can generate a Guestbook Response once and just replace the DataFile for each file in the list + DataFile firstDatafile = datafilesMap.values().size() > 0 ? (DataFile) Arrays.stream(datafilesMap.values().toArray()).findFirst().get() : null; + GuestbookResponse gbr = getGuestbookResponseFromBody(firstDatafile, GuestbookResponse.DOWNLOAD, jsonBody, user); + for (DataFile df : datafilesMap.values()) { + displayName = df.getDisplayName(); + if (checkGuestbookRequiredResponse(crc, df)) { if (gbr != null) { - engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); + gbr.setDataFile(df); + guestbookResponseService.save(gbr); + MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); + mdcLogService.logEntry(entry); } else { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } } else if (gbrecs != true && df.isReleased()) { // Write Guestbook record if not done previously and file is released - guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, user); - gbrecs = true; // prevent it from being written again + GuestbookResponse defaultResponse = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, user); + guestbookResponseService.save(defaultResponse); + MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); + mdcLogService.logEntry(entry); } - } catch (JsonParseException | CommandException ex) { - List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } + } catch (JsonParseException ex) { + List args = Arrays.asList(displayName, ex.getLocalizedMessage()); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } - return signed ? returnSignedUrl(datafilesMap, uriInfo, user, gbrecs) : null; + return returnSignedUrl(uriInfo, user); } private Map getDatafilesMap(ContainerRequestContext crc, String fileIds) { @@ -461,26 +465,30 @@ private Map getDatafilesMap(ContainerRequestContext crc, String return datafilesMap; } - private Response returnSignedUrl(Map datafilesMap, UriInfo uriInfo, User user, boolean gbrecs) { - AuthenticatedUser requestor = (AuthenticatedUser) user; + private Response returnSignedUrl(UriInfo uriInfo, User user) { // Create the signed URL - if (!datafilesMap.isEmpty()) { - UriBuilder builder = UriBuilder.fromUri(uriInfo.getRequestUri()); - builder.replaceQueryParam("gbrecs", String.valueOf(gbrecs)); - URI modifiedUri = builder.build(); - - String baseUrlEncoded = modifiedUri.toString(); - String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); - String key = ""; + String userIdentifier = null; + String key = null; + if (user != null && user instanceof AuthenticatedUser) { + AuthenticatedUser requestor = (AuthenticatedUser) user; + userIdentifier = requestor.getUserIdentifier(); ApiToken apiToken = authSvc.findApiTokenByUser(requestor); if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { key = apiToken.getTokenString(); } - String signedUrl = UrlSignerUtil.signUrl(baseUrl, 10, requestor.getUserIdentifier(), "GET", key); - return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); } else { - return notFound("no file ids were given"); - } + // Guest + userIdentifier = "guest"; + key = uriInfo.getAbsolutePath().toASCIIString(); //TODO find a better one for here and in SignedUrlAuthMechanism.java + } + + UriBuilder builder = UriBuilder.fromUri(uriInfo.getRequestUri()); + builder.replaceQueryParam("gbrecs", true); + String baseUrlEncoded = builder.build().toString(); + String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); + key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; + String signedUrl = UrlSignerUtil.signUrl(baseUrl, 1, userIdentifier, "GET", key); + return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); } /* @@ -716,9 +724,11 @@ public DownloadInstance downloadAuxiliaryFile(@Context ContainerRequestContext c @Produces({ "application/zip" }) public Response postDownloadDatafiles(@Context ContainerRequestContext crc, String body, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - Response res = processDatafileWithGuestbookResponse(crc, body, uriInfo, gbrecs, false, body); + + Response res = processDatafileWithGuestbookResponse(crc, headers, body, uriInfo, gbrecs, body); if (res != null) { - return res; // must be an error since signed url is not an option + return res; + // TODO: There is no get for this so we shouldn't return a signed url } else { // initiate the download now return downloadDatafiles(crc, body, gbrecs, uriInfo, headers, response, null); @@ -729,7 +739,7 @@ public Response postDownloadDatafiles(@Context ContainerRequestContext crc, Stri @AuthRequired @Path("dataset/{id}") @Produces({"application/zip"}) - public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { try { User user = getRequestUser(crc); @@ -744,11 +754,7 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat // We don't want downloads from Draft versions to be counted, // so we are setting the gbrecs (aka "do not write guestbook response") // variable accordingly: - if (signed) { - return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, true); - } else { - return downloadDatafiles(crc, fileIds, true, uriInfo, headers, response, "draft"); - } + return downloadDatafiles(crc, fileIds, true, uriInfo, headers, response, "draft"); } } @@ -769,11 +775,7 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat } String fileIds = getFileIdsAsCommaSeparated(latest.getFileMetadatas()); - if (signed) { - return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, gbrecs); - } else { - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, latest.getFriendlyVersionNumber()); - } + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, latest.getFriendlyVersionNumber()); } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -782,7 +784,7 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat @AuthRequired @Path("dataset/{id}") @Produces({"application/zip"}) - public Response downloadAllFromLatestWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) throws WebApplicationException { + public Response downloadAllFromLatestWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) throws WebApplicationException { try { User user = getRequestUser(crc); DataverseRequest req = createDataverseRequest(user); @@ -803,14 +805,7 @@ public Response downloadAllFromLatestWithGuestbookResponse(@Context ContainerReq fileIds = getFileIdsAsCommaSeparated(latest.getFileMetadatas()); version = latest.getFriendlyVersionNumber(); } - Response res = processDatafileWithGuestbookResponse(crc, fileIds, uriInfo, gbrecs, signed, jsonBody); - - if (res != null) { - return res; // could be an error or a signedUrl in the response - } else { - // initiate the download now - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, version); - } + return processDatafileWithGuestbookResponse(crc, headers, fileIds, uriInfo, gbrecs, jsonBody); } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -837,12 +832,8 @@ public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @Pa if (dsv.isDraft()) { gbrecs = true; } - if (signed) { - AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); - return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, gbrecs); - } else { - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, dsv.getFriendlyVersionNumber().toLowerCase()); - } + + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, dsv.getFriendlyVersionNumber().toLowerCase()); } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -853,18 +844,12 @@ public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @Pa @Path("dataset/{id}/versions/{versionId}") @Produces({"application/zip"}) public Response downloadAllFromVersionWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @PathParam("versionId") String versionId, - @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @QueryParam("signed") Boolean signed, String jsonBody, + @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, String jsonBody, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { try { DatasetVersion dsv = getDatasetVersionFromVersion(crc, datasetIdOrPersistentId, versionId); String fileIds = getFileIdsAsCommaSeparated(dsv.getFileMetadatas()); - Response res = processDatafileWithGuestbookResponse(crc, fileIds, uriInfo, gbrecs, signed, jsonBody); - if (res != null) { - return res; // could be an error or a signedUrl in the response - } else { - // initiate the download now - return downloadAllFromVersion(crc, datasetIdOrPersistentId, versionId, gbrecs, apiTokenParam, false, uriInfo, headers, response); - } + return processDatafileWithGuestbookResponse(crc, headers, fileIds, uriInfo, gbrecs, jsonBody); } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -931,30 +916,20 @@ private String generateMultiFileBundleName(Dataset dataset, String versionTag) { @AuthRequired @Path("datafiles/{fileIds}") @Produces({"application/zip"}) - public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - if (signed) { - AuthenticatedUser user = (AuthenticatedUser) getRequestUser(crc); - return returnSignedUrl(getDatafilesMap(crc, fileIds), uriInfo, user, gbrecs); - } else { - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); - } + + return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); } @POST @AuthRequired @Path("datafiles/{fileIds}") @Produces({"application/zip"}) - public Response datafilesWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("signed") boolean signed, + public Response datafilesWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) throws WebApplicationException { - Response res = processDatafileWithGuestbookResponse(crc, fileIds, uriInfo, gbrecs, signed, jsonBody); - if (res != null) { - return res; // could be an error or a signedUrl in the response - } else { - // initiate the download now - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); - } + return processDatafileWithGuestbookResponse(crc, headers, fileIds, uriInfo, gbrecs, jsonBody); } private String[] getFileIdsCSV(String body) { @@ -1025,7 +1000,7 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo if (datasetIds.size() > 1) { // All files must be from the same Dataset return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.multipleDatasets")); - } else if (authorizedDatafileIds.contains(df.getId()) && checkGuestbookRequiredResponse(user, df)) { + } else if (authorizedDatafileIds.contains(df.getId()) && checkGuestbookRequiredResponse(crc, df)) { try { GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, body, user); if (gbr != null) { @@ -1953,39 +1928,22 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @ return ok(jsonObjectBuilder); } - private boolean checkGuestbookRequiredResponse(User user, DataFile df) throws WebApplicationException { - // Check if guestbook response is required and one does not already exist - boolean required = false; - if (df.isRestricted() && df.getOwner().hasEnabledGuestbook()) { - required = true; - // if we find an existing response for this user/datafile then it is not required to add another one - List gbrList = user instanceof AuthenticatedUser ? guestbookResponseService.findByAuthenticatedUserId((AuthenticatedUser)user) : null; - if (gbrList != null) { - // no need to check for nulls since if it's enabled it must exist - final Long guestbookId = df.getOwner().getGuestbook().getId(); - - // find a matching response for the datafile/guestbook combination - // this forces a new response if the guestbook changed - for (GuestbookResponse r : gbrList) { - if (df.getId().equals(r.getDataFile().getId()) && guestbookId.equals(r.getGuestbook().getId())) { - required = false; - break; - } - } - } - } + private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df) { + // Check if guestbook response is required + // check if this was originally a signed url - since the property only exists if true then a null check should suffice + Object wasSigned = crc.getProperty("wasSigned"); + boolean required = wasSigned == null && df.getOwner().hasEnabledGuestbook(); return required; } private GuestbookResponse getGuestbookResponseFromBody(DataFile dataFile, String type, String jsonBody, User requestor) throws JsonParseException { - Dataset ds = dataFile.getOwner(); GuestbookResponse guestbookResponse = null; - if (jsonBody != null && jsonBody.startsWith("{")) { + if (dataFile != null && jsonBody != null && jsonBody.startsWith("{")) { JsonObject guestbookResponseObj = JsonUtil.getJsonObject(jsonBody).getJsonObject("guestbookResponse"); - guestbookResponse = guestbookResponseService.initAPIGuestbookResponse(ds, dataFile, null, requestor); + guestbookResponse = guestbookResponseService.initAPIGuestbookResponse(dataFile.getOwner(), dataFile, null, requestor); guestbookResponse.setEventType(type); - // Parse custom question answers + // Parse custom question answers and validate them jsonParser().parseGuestbookResponse(guestbookResponseObj, guestbookResponse); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java index e701876d5ce..258770da0d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java @@ -1,10 +1,7 @@ package edu.harvard.iq.dataverse.api.auth; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; -import edu.harvard.iq.dataverse.authorization.users.ApiToken; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; -import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.authorization.users.*; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.settings.JvmSettings; @@ -56,15 +53,20 @@ private String getSignedUrlRequestParameter(ContainerRequestContext containerReq private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext containerRequestContext) { User user = null; // The signedUrl contains a param telling which user this is supposed to be for. - // We don't trust this. So we lookup that user, and get their API key, and use + // We don't trust this. So we look up that user, and get their API key, and use // that as a secret in validating the signedURL. If the signature can't be - // validated with their key, the user (or their API key) has been changed and + // validated with their key, the user (or their API key) has been changed, and // we reject the request. + // If User is Guest we can return a generic guest user with key made from URI UriInfo uriInfo = containerRequestContext.getUriInfo(); String userId = uriInfo.getQueryParameters().getFirst(SIGNED_URL_USER); - User targetUser = null; + User targetUser = null; ApiToken userApiToken = null; - if (!userId.startsWith(PrivateUrlUser.PREFIX)) { + if (userId.equalsIgnoreCase("guest")) { + targetUser = GuestUser.get(); + userApiToken = new ApiToken(); + userApiToken.setTokenString(uriInfo.getAbsolutePath().toASCIIString()); //TODO find a better one for here and in Access.java + } else if (!userId.startsWith(PrivateUrlUser.PREFIX)) { targetUser = authSvc.getAuthenticatedUser(userId); userApiToken = authSvc.findApiTokenByUser((AuthenticatedUser) targetUser); } else { @@ -92,6 +94,7 @@ private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext container boolean isSignedUrlValid = UrlSignerUtil.isValidUrl(signedUrl, userId, requestMethod, signedUrlSigningKey); if (isSignedUrlValid) { user = targetUser; + containerRequestContext.setProperty("wasSigned", Boolean.TRUE); } } return user; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 2cba4faceb0..7de50ae0d0b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -19,6 +19,7 @@ import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; @@ -617,8 +618,23 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons } guestbookResponse.setInstitution(obj.getString("institution", guestbookResponse.getInstitution())); guestbookResponse.setPosition(obj.getString("position", guestbookResponse.getPosition())); + Guestbook guestbook = guestbookResponse.getGuestbook(); + List missingResponses = new ArrayList<>(); + if (guestbook.isNameRequired() && StringUtil.isEmpty(guestbookResponse.getName())) { + missingResponses.add("Name"); + } + if (guestbook.isEmailRequired() && StringUtil.isEmpty(guestbookResponse.getEmail())) { + missingResponses.add("Email"); + } + if (guestbook.isInstitutionRequired() && StringUtil.isEmpty(guestbookResponse.getInstitution())) { + missingResponses.add("Institution"); + } + if (guestbook.isPositionRequired() && StringUtil.isEmpty(guestbookResponse.getPosition())) { + missingResponses.add("Position"); + } + Map cqMap = new HashMap<>(); - guestbookResponse.getGuestbook().getCustomQuestions().stream().forEach(cq -> cqMap.put(cq.getId(),cq)); + guestbook.getCustomQuestions().stream().forEach(cq -> cqMap.put(cq.getId(),cq)); JsonArray answers = obj.getJsonArray("answers"); List customQuestionResponses = new ArrayList<>(); for (JsonObject answer : answers.getValuesAs(JsonObject.class)) { @@ -651,7 +667,6 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons } guestbookResponse.setCustomQuestionResponses(customQuestionResponses); // verify each required question is in the response - List missingResponses = new ArrayList<>(); for (Map.Entry e : cqMap.entrySet()) { if (e.getValue().isRequired()) { missingResponses.add(e.getValue().getQuestionString()); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index cc334bd2782..08cb99b9901 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2927,7 +2927,7 @@ access.api.requestAccess.failure.requestExists=An access request for this file o access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. access.api.requestAccess.failure.guestbookAccessRequestResponseMissing=You may not request access to this file without the required Guestbook response. -access.api.requestAccess.failure.guestbookresponseMissingRequired=Guestbook Custom Question Answer is required but not present ({0}). +access.api.requestAccess.failure.guestbookresponseMissingRequired=Guestbook Response entry is required but not present ({0}). access.api.requestAccess.failure.guestbookresponseInvalidOption=Guestbook Custom Question Answer not a valid option ({0}). access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound=Guestbook Custom Question ID {0} not found. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 4305981a28d..26ee900e91b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3929,6 +3929,10 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/cc0.png", json1.build(), ownerApiToken); uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); Integer fileId3 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); + JsonObjectBuilder json4 = Json.createObjectBuilder().add("description", "my description4").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); + uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/Robot-Icon_2.png", json1.build(), ownerApiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + Integer fileId4 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); // Restrict files Response restrictResponse = UtilIT.restrictFile(fileId1.toString(), true, ownerApiToken); @@ -3937,6 +3941,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); restrictResponse = UtilIT.restrictFile(fileId3.toString(), true, ownerApiToken); restrictResponse.then().assertThat().statusCode(OK.getStatusCode()); + // do not restrict fileId4 // Update Dataset to allow requests Response allowAccessRequestsResponse = UtilIT.allowAccessRequests(datasetId.toString(), true, ownerApiToken); @@ -3965,8 +3970,35 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars String guestbookResponse = UtilIT.generateGuestbookResponse(guestbook); + // Download unrestricted file by guest user fails without GuestbookResponse + Response downloadResponse = UtilIT.downloadFile(fileId4); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .body("status", equalTo(ApiConstants.STATUS_ERROR)) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) + .statusCode(BAD_REQUEST.getStatusCode()); + // With GuestbookResponse. Guest user doesn't have the required Name and Email. so this will still fail + downloadResponse = UtilIT.postDownloadFile(fileId4, guestbookResponse); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .body("status", equalTo(ApiConstants.STATUS_ERROR)) + .body("message", containsString("(Name,Email)")) + .statusCode(BAD_REQUEST.getStatusCode()); + String guestbookResponseForGuest = guestbookResponse.replace("\"guestbookResponse\": {", + "\"guestbookResponse\": { \"name\":\"My Name\", \"email\":\"myemail@example.com\", \"position\":\"My Position\", \"institution\":\"My Institution\","); + // With GuestbookResponse. Guest user doesn't have the required Name, etc. So we will add those to the Guestbook Response + downloadResponse = UtilIT.postDownloadFile(fileId4, guestbookResponseForGuest); + downloadResponse.prettyPrint(); + downloadResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + String signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); + // Download the file using the signed url + Response signedUrlResponse = get(signedUrl); + signedUrlResponse.prettyPrint(); + assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); + // Get Download Url attempt - Guestbook Response is required but not found - Response downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken, null, false); + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken, null); downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) @@ -3974,14 +4006,14 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars .statusCode(BAD_REQUEST.getStatusCode()); // Get Signed Download Url with guestbook response - downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken, guestbookResponse, true); + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken, guestbookResponse); downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .statusCode(OK.getStatusCode()); - String signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); + signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); // Download the file using the signed url - Response signedUrlResponse = get(signedUrl); + signedUrlResponse = get(signedUrl); signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); @@ -3999,7 +4031,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars downloadResponse.prettyPrint(); assertEquals(OK.getStatusCode(), downloadResponse.getStatusCode()); - downloadResponse = UtilIT.downloadFilesUrlWithGuestbookResponse(new Integer[]{fileId1, fileId2, fileId3}, apiToken, guestbookResponse, true); + downloadResponse = UtilIT.downloadFilesUrlWithGuestbookResponse(new Integer[]{fileId1, fileId2, fileId3}, apiToken, guestbookResponse); downloadResponse.prettyPrint(); signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); signedUrlResponse = get(signedUrl); @@ -4014,7 +4046,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars // Modify guestbookResponse excluding email to show that the email remains unchanged guestbookResponse = guestbookResponse.replace("\"guestbookResponse\": {", "\"guestbookResponse\": { \"name\":\"My Name\", \"position\":\"My Position\", \"institution\":\"My Institution\","); - downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken2, guestbookResponse, true); + downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken2, guestbookResponse); downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .statusCode(OK.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e4fb8e1053e..5577daaed27 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1201,7 +1201,11 @@ static Response downloadFile(Integer fileId) { // .header(API_TOKEN_HTTP_HEADER, apiToken) .get("/api/access/datafile/" + fileId); } - + static Response postDownloadFile(Integer fileId, String jsonBody) { + return given() + .body(jsonBody) + .post("/api/access/datafile/" + fileId); + } static Response downloadFile(Integer fileId, String apiToken) { String nullByteRange = null; String nullFormat = null; @@ -1259,20 +1263,18 @@ static Response downloadFileOriginal(Integer fileId, String apiToken) { .get("/api/access/datafile/" + fileId + "?format=original&key=" + apiToken); } - static Response getDownloadFileUrlWithGuestbookResponse(Integer fileId, String apiToken, String body, boolean signed) { + static Response getDownloadFileUrlWithGuestbookResponse(Integer fileId, String apiToken, String body) { RequestSpecification requestSpecification = given(); requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); - String signedParam = signed ? "?signed=true" : ""; if (body != null) { requestSpecification.body(body); } - return requestSpecification.post("/api/access/datafile/" + fileId + signedParam); + return requestSpecification.post("/api/access/datafile/" + fileId); } - static Response downloadFilesUrlWithGuestbookResponse(Integer[] fileIds, String apiToken, String body, boolean signed) { + static Response downloadFilesUrlWithGuestbookResponse(Integer[] fileIds, String apiToken, String body) { RequestSpecification requestSpecification = given(); requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); - String signedParam = signed ? "?signed=true" : ""; if (body != null) { requestSpecification.body(body); } @@ -1280,7 +1282,7 @@ static Response downloadFilesUrlWithGuestbookResponse(Integer[] fileIds, String for (Integer fileId : fileIds) { getString += fileId + ","; } - return requestSpecification.post(getString + signedParam); + return requestSpecification.post(getString); } static Response postDownloadDatafiles(String body, String apiToken) { From c9831ada513798778ec4ed59ef191a881d029d3e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 4 Mar 2026 15:07:55 -0500 Subject: [PATCH 27/52] remove signed= from docs --- doc/release-notes/12001-api-support-termofuse-guestbook.md | 6 +----- doc/sphinx-guides/source/api/dataaccess.rst | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/12001-api-support-termofuse-guestbook.md b/doc/release-notes/12001-api-support-termofuse-guestbook.md index daa3ddc63a2..249aabd2902 100644 --- a/doc/release-notes/12001-api-support-termofuse-guestbook.md +++ b/doc/release-notes/12001-api-support-termofuse-guestbook.md @@ -1,17 +1,13 @@ ## Feature Request: API to support Download Terms of Use and Guestbook ## New Endpoints to download a file or files that required a Guestbook Response: POST -A post to these endpoints with the body containing a JSON Guestbook Response will save the response and -`?signed=true`: return a signed URL to download the file(s) or -`?signed=false` or missing: Write the Guestbook Responses and download the file(s) +A post to these endpoints with the body containing a JSON Guestbook Response will save the response and return a signed URL to download the file(s) `/api/access/datafile/{fileId:.+}` `/api/access/datafiles/{fileIds}` `/api/access/dataset/{id}` `/api/access/dataset/{id}/versions/{versionId}` -The matching GET APIs will also take the `?signed=true` parameter to also return the signed url instead of downloading immediately. Note: Signed urls are only for Authenticated Users. Guest users will receive an error if requesting with signed=true - A post to these endpoints with the body containing a JSON Guestbook Response will save the response before continuing the download. No signed URL option exists. `/api/access/datafiles` diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index e52f2c4fbab..01431a02345 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,7 +32,7 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (no version): @@ -61,7 +61,7 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (with version): @@ -95,7 +95,7 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url (with query parameter ``&signed=true``) that can be used to download the file(s) via a browser or download manager. Without the ``signed`` parameter the download will start immediately. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. +.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Example :: From bc06e3b99e1f2bf9dfd0585df9deae47ec929b36 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:37:48 -0500 Subject: [PATCH 28/52] fix getting file ids from csv --- .../edu/harvard/iq/dataverse/api/Access.java | 94 +++++++++---------- 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index e759b95e2dc..d4ca8922dd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -59,6 +59,7 @@ import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; @@ -198,14 +199,10 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr @Produces({"application/zip"}) public BundleDownloadInstance datafileBundleWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - Response res = processDatafileWithGuestbookResponse(crc, headers, fileId, uriInfo, gbrecs, jsonBody); - if (res != null) { - throw new WebApplicationException(res); - // TODO: There is no get for this so we shouldn't return a signed url - } else { - // return the download instance - return datafileBundle(crc, fileId, fileMetadataId, gbrecs, uriInfo, headers, response); - } + processDatafileWithGuestbookResponse(crc, headers, fileId, uriInfo, gbrecs, jsonBody); + // There is no get for this so we shouldn't return a signed url + // return the download instance + return datafileBundle(crc, fileId, fileMetadataId, gbrecs, uriInfo, headers, response); } //Added a wrapper method since the original method throws a wrapped response @@ -724,15 +721,10 @@ public DownloadInstance downloadAuxiliaryFile(@Context ContainerRequestContext c @Produces({ "application/zip" }) public Response postDownloadDatafiles(@Context ContainerRequestContext crc, String body, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - - Response res = processDatafileWithGuestbookResponse(crc, headers, body, uriInfo, gbrecs, body); - if (res != null) { - return res; - // TODO: There is no get for this so we shouldn't return a signed url - } else { - // initiate the download now - return downloadDatafiles(crc, body, gbrecs, uriInfo, headers, response, null); - } + processDatafileWithGuestbookResponse(crc, headers, body, uriInfo, gbrecs, body); + // There is no get for this so we shouldn't return a signed url + // initiate the download now + return downloadDatafiles(crc, body, gbrecs, uriInfo, headers, response, null); } @GET @@ -938,9 +930,7 @@ private String[] getFileIdsCSV(String body) { "fileIds=1,2,3" {fileIds:[1,2,3], "guestbookResponse":{}} */ - if (body.startsWith("fileIds=")) { - return body.substring(8).split(","); // Trim string "fileIds=" from the front - } else if (body.startsWith("{")) { // assume json + if (body.startsWith("{")) { // assume json // get fileIds from json. example: {fileIds:[1,2,3], "guestbookResponse":{}} JsonObject jsonObject = JsonUtil.getJsonObject(body); if (jsonObject.containsKey("fileIds")) { @@ -951,8 +941,11 @@ private String[] getFileIdsCSV(String body) { return new String[0]; } } else { - // default to expected list of ids "1,2,3" - return body.split(","); + // Trim string "fileIds=" from the front if exists + String csv = body.substring(body.startsWith("fileIds=") ? 8 : 0); + //String[] list = body.substring(body.startsWith("fileIds=") ? 8 : 0).replaceAll(",$", "").split(","); + return Arrays.asList(csv.split(",")).stream().map(String::trim) + .filter(s -> !s.isEmpty()).collect(Collectors.toList()).toArray(new String[0]); } } @@ -990,33 +983,31 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo // Get DataFiles, check authorized access, check for multiple Datasets, and check for required guestbook response Set datasetIds = new HashSet<>(); for (int i = 0; i < fileIdParams.length; i++) { - if (!fileIdParams[i].isBlank()) { - DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); - datafilesMap.put(df.getId(), df); - datasetIds.add(df.getOwner() != null ? df.getOwner().getId() : 0L); - if (isAccessAuthorized(user, df)) { - authorizedDatafileIds.add(df.getId()); - } - if (datasetIds.size() > 1) { - // All files must be from the same Dataset - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.multipleDatasets")); - } else if (authorizedDatafileIds.contains(df.getId()) && checkGuestbookRequiredResponse(crc, df)) { - try { - GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, body, user); - if (gbr != null) { - engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); - donotwriteGBResponse = true; - // Further down the actual download will also create a simple download response for every datafile listed based on the donotwriteGBResponse flag. - // Modifying donotwriteGBResponse will block that so we also need to log the MDC entry here - MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); - mdcLogService.logEntry(entry); - } else { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); - } - } catch (JsonParseException | CommandException ex) { - List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); + DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); + datafilesMap.put(df.getId(), df); + datasetIds.add(df.getOwner() != null ? df.getOwner().getId() : 0L); + if (isAccessAuthorized(user, df)) { + authorizedDatafileIds.add(df.getId()); + } + if (datasetIds.size() > 1) { + // All files must be from the same Dataset + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.multipleDatasets")); + } else if (authorizedDatafileIds.contains(df.getId()) && checkGuestbookRequiredResponse(crc, df)) { + try { + GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, body, user); + if (gbr != null) { + engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), gbr, gbr.getDataset())); + donotwriteGBResponse = true; + // Further down the actual download will also create a simple download response for every datafile listed based on the donotwriteGBResponse flag. + // Modifying donotwriteGBResponse will block that so we also need to log the MDC entry here + MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); + mdcLogService.logEntry(entry); + } else { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } + } catch (JsonParseException | CommandException ex) { + List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } } } @@ -1931,9 +1922,10 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @ private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df) { // Check if guestbook response is required // check if this was originally a signed url - since the property only exists if true then a null check should suffice - Object wasSigned = crc.getProperty("wasSigned"); - boolean required = wasSigned == null && df.getOwner().hasEnabledGuestbook(); - return required; + boolean notSigned = (crc.getProperty("wasSigned") == null); + boolean wasWrittenInPost = true; // TODO: not required if signed and has some flag stating that the guestbook response was already written + + return notSigned && !wasWrittenInPost && df.getOwner().hasEnabledGuestbook(); } private GuestbookResponse getGuestbookResponseFromBody(DataFile dataFile, String type, String jsonBody, User requestor) throws JsonParseException { From 0f8bd2889a1f6ec0ab767f4368f4202fff715a24 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:48:43 -0500 Subject: [PATCH 29/52] fixing questbook response handling --- .../harvard/iq/dataverse/DatasetVersion.java | 54 +++------- .../iq/dataverse/FileDownloadServiceBean.java | 65 ++++++------ .../edu/harvard/iq/dataverse/api/Access.java | 98 +++++++++++++------ .../api/auth/SignedUrlAuthMechanism.java | 1 - .../harvard/iq/dataverse/util/FileUtil.java | 95 ++++++------------ .../iq/dataverse/util/FileUtilTest.java | 39 ++++---- 6 files changed, 159 insertions(+), 193 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 71f5c229e3d..cca57b367cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1,21 +1,22 @@ package edu.harvard.iq.dataverse; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.MarkupChecker; -import edu.harvard.iq.dataverse.util.PersonOrOrgUtil; -import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.DataFileComparator; import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.license.License; -import edu.harvard.iq.dataverse.util.FileUtil; -import edu.harvard.iq.dataverse.util.StringUtil; -import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.DateUtil; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.*; import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import edu.harvard.iq.dataverse.workflows.WorkflowComment; +import jakarta.json.*; +import jakarta.persistence.*; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import jakarta.validation.constraints.Size; +import org.apache.commons.lang3.StringUtils; + import java.io.Serializable; import java.net.MalformedURLException; import java.net.URI; @@ -28,39 +29,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Index; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.NamedQueries; -import jakarta.persistence.NamedQuery; -import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; -import jakarta.persistence.OrderBy; -import jakarta.persistence.Table; -import jakarta.persistence.Temporal; -import jakarta.persistence.TemporalType; -import jakarta.persistence.Transient; -import jakarta.persistence.UniqueConstraint; -import jakarta.persistence.Version; -import jakarta.validation.ConstraintViolation; -import jakarta.validation.Validation; -import jakarta.validation.Validator; -import jakarta.validation.constraints.Size; -import org.apache.commons.lang3.StringUtils; - /** * * @author skraffmiller @@ -2131,7 +2099,7 @@ public String getJsonLd() { boolean hideFilesBoolean = JvmSettings.HIDE_SCHEMA_DOT_ORG_DOWNLOAD_URLS.lookupOptional(Boolean.class).orElse(false); if (!hideFilesBoolean) { String nullDownloadType = null; - fileObject.add("contentUrl", dataverseSiteUrl + FileUtil.getFileDownloadUrlPath(nullDownloadType, fileMetadata.getDataFile().getId(), false, fileMetadata.getId())); + fileObject.add("contentUrl", dataverseSiteUrl + FileUtil.getFileDownloadUrlPath(nullDownloadType, fileMetadata.getDataFile().getId(), false, fileMetadata.getId(), null)); } fileArray.add(fileObject); } diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index 5370e9ac564..ce8e8b5f4bc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -7,7 +7,6 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookResponseCommand; import edu.harvard.iq.dataverse.engine.command.impl.RequestAccessCommand; @@ -16,20 +15,7 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.FileUtil; -import edu.harvard.iq.dataverse.util.JsfHelper; -import edu.harvard.iq.dataverse.util.StringUtil; -import edu.harvard.iq.dataverse.util.URLTokenUtil; - -import java.io.IOException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.UUID; -import java.util.logging.Logger; +import edu.harvard.iq.dataverse.util.*; import jakarta.ejb.EJB; import jakarta.ejb.Stateless; import jakarta.faces.context.FacesContext; @@ -37,11 +23,14 @@ import jakarta.inject.Named; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; -import jakarta.persistence.Query; import jakarta.servlet.ServletOutputStream; import jakarta.servlet.http.HttpServletResponse; - import org.primefaces.PrimeFaces; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.*; +import java.util.logging.Logger; //import org.primefaces.context.RequestContext; /** @@ -104,6 +93,7 @@ public void writeGuestbookAndStartBatchDownload(GuestbookResponse guestbookRespo } boolean original = "original".equals(guestbookResponse.getFileFormat()); + String gbrid = null; // Let's intercept the case where a multiple download method was called, // with only 1 file on the list. We'll treat it like a single file download @@ -122,10 +112,10 @@ public void writeGuestbookAndStartBatchDownload(GuestbookResponse guestbookRespo if (!doNotSaveGuestbookRecord) { DataFile df = datafileService.findCheapAndEasy(Long.parseLong(fileIds[0])); guestbookResponse.setDataFile(df); - writeGuestbookResponseRecord(guestbookResponse); + gbrid = writeGuestbookResponseRecord(guestbookResponse); } - redirectToDownloadAPI(guestbookResponse.getFileFormat(), fileId, true, null); + redirectToDownloadAPI(guestbookResponse.getFileFormat(), fileId, true, gbrid, null); return; } @@ -147,7 +137,7 @@ public void writeGuestbookAndStartBatchDownload(GuestbookResponse guestbookRespo if (df != null) { if (!doNotSaveGuestbookRecord) { guestbookResponse.setDataFile(df); - writeGuestbookResponseRecord(guestbookResponse); + gbrid = writeGuestbookResponseRecord(guestbookResponse); } if (useCustomZipService) { @@ -168,18 +158,18 @@ public void writeGuestbookAndStartBatchDownload(GuestbookResponse guestbookRespo redirectToCustomZipDownloadService(customZipDownloadUrl, zipServiceKey); } else { // Use the "normal" /api/access/datafiles/ API: - redirectToBatchDownloadAPI(guestbookResponse.getSelectedFileIds(),original); + redirectToBatchDownloadAPI(guestbookResponse.getSelectedFileIds(),original, gbrid); } } - public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format) { + public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format, String gbrIds) { if(!fileMetadata.getDatasetVersion().isDraft()){ guestbookResponse = guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, format); writeGuestbookResponseRecord(guestbookResponse); } // Make sure to set the "do not write Guestbook response" flag to TRUE when calling the Access API: - redirectToDownloadAPI(format, fileMetadata.getDataFile().getId(), true, fileMetadata.getId()); + redirectToDownloadAPI(format, fileMetadata.getDataFile().getId(), true, gbrIds, fileMetadata.getId()); logger.fine("issued file download redirect for filemetadata "+fileMetadata.getId()+", datafile "+fileMetadata.getDataFile().getId()); } @@ -188,9 +178,9 @@ public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookRespon logger.warning("writeGuestbookAndStartFileDownload(GuestbookResponse) called without the DataFile in the GuestbookResponse."); return; } - writeGuestbookResponseRecord(guestbookResponse); + String gbrId = writeGuestbookResponseRecord(guestbookResponse); - redirectToDownloadAPI(guestbookResponse.getFileFormat(), guestbookResponse.getDataFile().getId()); + redirectToDownloadAPI(guestbookResponse.getFileFormat(), guestbookResponse.getDataFile().getId(), gbrId); logger.fine("issued file download redirect for datafile "+guestbookResponse.getDataFile().getId()); } @@ -237,10 +227,12 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, Fi } } - public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) { + public String writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) { + String guestbookResponseIds = ""; try { CreateGuestbookResponseCommand cmd = new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset()); commandEngine.submit(cmd); + guestbookResponseIds = guestbookResponse.getId().toString(); DatasetVersion version = guestbookResponse.getDatasetVersion(); //Sometimes guestbookResponse doesn't have a version, so we grab the released version @@ -256,6 +248,7 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) { //if an error occurs here then download won't happen no need for response recs... logger.warning("Exception writing GuestbookResponse for file: " + guestbookResponse.getDataFile().getId() + " : " + e.getLocalizedMessage()); } + return guestbookResponseIds; } public void writeGuestbookResponseRecordForRequestAccess(GuestbookResponse guestbookResponse) { @@ -286,7 +279,7 @@ public void writeGuestbookResponseRecordForRequestAccess(GuestbookResponse guest // But note that this may change - there may be some future situations where it will // become necessary again, to pass the job of creating the access record // to the API. - private void redirectToBatchDownloadAPI(String multiFileString, Boolean guestbookRecordsAlreadyWritten, Boolean downloadOriginal){ + private void redirectToBatchDownloadAPI(String multiFileString, Boolean guestbookRecordsAlreadyWritten, Boolean downloadOriginal, String gbrIds){ String fileDownloadUrl = "/api/access/datafiles"; if (guestbookRecordsAlreadyWritten && !downloadOriginal){ @@ -296,7 +289,9 @@ private void redirectToBatchDownloadAPI(String multiFileString, Boolean guestboo } else if (!guestbookRecordsAlreadyWritten && downloadOriginal){ fileDownloadUrl += "?format=original"; } - + if (gbrIds != null && !gbrIds.isEmpty()) { + fileDownloadUrl += (fileDownloadUrl.contains("?") ? "&" : "?") + "gbrids=" + gbrIds; + } PrimeFaces.current().executeScript("downloadFiles('"+fileDownloadUrl + "','"+ multiFileString+"');"); } @@ -312,10 +307,8 @@ private void redirectToCustomZipDownloadService(String customZipServiceUrl, Stri } } - private void redirectToDownloadAPI(String downloadType, Long fileId, boolean guestBookRecordAlreadyWritten, - Long fileMetadataId) { - String fileDownloadUrl = FileUtil.getFileDownloadUrlPath(downloadType, fileId, guestBookRecordAlreadyWritten, - fileMetadataId); + private void redirectToDownloadAPI(String downloadType, Long fileId, boolean guestBookRecordAlreadyWritten, String gbrIds, Long fileMetadataId) { + String fileDownloadUrl = FileUtil.getFileDownloadUrlPath(downloadType, fileId, guestBookRecordAlreadyWritten, fileMetadataId, gbrIds); if ("GlobusTransfer".equals(downloadType)) { PrimeFaces.current().executeScript(URLTokenUtil.getScriptForUrl(fileDownloadUrl)); } else { @@ -328,12 +321,12 @@ private void redirectToDownloadAPI(String downloadType, Long fileId, boolean gue } } - private void redirectToDownloadAPI(String downloadType, Long fileId) { - redirectToDownloadAPI(downloadType, fileId, true, null); + private void redirectToDownloadAPI(String downloadType, Long fileId, String gbrIds) { + redirectToDownloadAPI(downloadType, fileId, true, gbrIds, null); } - private void redirectToBatchDownloadAPI(String multiFileString, Boolean downloadOriginal){ - redirectToBatchDownloadAPI(multiFileString, true, downloadOriginal); + private void redirectToBatchDownloadAPI(String multiFileString, Boolean downloadOriginal, String gbrIds){ + redirectToBatchDownloadAPI(multiFileString, true, downloadOriginal, gbrIds); } public void redirectToAuxFileDownloadAPI(Long fileId, String formatTag, String formatVersion) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index d4ca8922dd5..e53b5079f4b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -56,6 +56,8 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.sql.Timestamp; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; @@ -136,20 +138,20 @@ public class Access extends AbstractApiBean { @Path("datafile/bundle/{fileId}") @Produces({"application/zip"}) public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, - @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, + @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { DataFile df = findDataFileOrDieWrapper(fileId); // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); - - if (checkGuestbookRequiredResponse(crc, df)) { + User requestor = getRequestor(crc); + if (checkGuestbookRequiredResponse(crc, df, gbrids)) { throw new BadRequestException(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } if (gbrecs != true && df.isReleased()) { // Write Guestbook record if not done previously and file is released - User requestor = getRequestor(crc); GuestbookResponse gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, requestor); guestbookResponseService.save(gbr); MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); @@ -197,12 +199,17 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr @AuthRequired @Path("datafile/bundle/{fileId}") @Produces({"application/zip"}) - public BundleDownloadInstance datafileBundleWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, @QueryParam("gbrecs") boolean gbrecs, + public BundleDownloadInstance datafileBundleWithGuestbookResponse(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("fileMetadataId") Long fileMetadataId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, String jsonBody) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + processDatafileWithGuestbookResponse(crc, headers, fileId, uriInfo, gbrecs, jsonBody); + // JSF UI passes the guestbook response id(s) in thus this qp can be removed when JSF is removed + if (gbrids == null || gbrids.isEmpty()) { + gbrids = (String) crc.getProperty("gbrids"); + } // There is no get for this so we shouldn't return a signed url // return the download instance - return datafileBundle(crc, fileId, fileMetadataId, gbrecs, uriInfo, headers, response); + return datafileBundle(crc, fileId, fileMetadataId, gbrecs, gbrids, uriInfo, headers, response); } //Added a wrapper method since the original method throws a wrapped response @@ -227,7 +234,7 @@ private DataFile findDataFileOrDieWrapper(String fileId){ @AuthRequired @Path("datafile/{fileId:.+}") @Produces({"application/xml","*/*"}) - public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, + public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { fileId = normalizeFileId(fileId); @@ -243,13 +250,14 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); - if (checkGuestbookRequiredResponse(crc, df)) { + User requestor = getRequestor(crc); + if (checkGuestbookRequiredResponse(crc, df, gbrids)) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } if (gbrecs != true && df.isReleased()){ // Write Guestbook record if not done previously and file is released - gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, getRequestor(crc)); + gbr = guestbookResponseService.initAPIGuestbookResponse(df.getOwner(), df, session, requestor); } DownloadInfo dInfo = new DownloadInfo(df); @@ -409,16 +417,19 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr // Handle Guestbook Responses String displayName = ""; + String gbrids = ""; try { // since all files must be in the same Dataset we can generate a Guestbook Response once and just replace the DataFile for each file in the list DataFile firstDatafile = datafilesMap.values().size() > 0 ? (DataFile) Arrays.stream(datafilesMap.values().toArray()).findFirst().get() : null; GuestbookResponse gbr = getGuestbookResponseFromBody(firstDatafile, GuestbookResponse.DOWNLOAD, jsonBody, user); + boolean guestbookResponseRequired = checkGuestbookRequiredResponse(crc, firstDatafile, null); for (DataFile df : datafilesMap.values()) { displayName = df.getDisplayName(); - if (checkGuestbookRequiredResponse(crc, df)) { + if (guestbookResponseRequired) { if (gbr != null) { gbr.setDataFile(df); guestbookResponseService.save(gbr); + gbrids = gbr.getId().toString(); MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); mdcLogService.logEntry(entry); } else { @@ -436,7 +447,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr List args = Arrays.asList(displayName, ex.getLocalizedMessage()); return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } - return returnSignedUrl(uriInfo, user); + return returnSignedUrl(crc, uriInfo, user, gbrids); } private Map getDatafilesMap(ContainerRequestContext crc, String fileIds) { @@ -462,7 +473,7 @@ private Map getDatafilesMap(ContainerRequestContext crc, String return datafilesMap; } - private Response returnSignedUrl(UriInfo uriInfo, User user) { + private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, User user, String gbrids) { // Create the signed URL String userIdentifier = null; String key = null; @@ -481,6 +492,8 @@ private Response returnSignedUrl(UriInfo uriInfo, User user) { UriBuilder builder = UriBuilder.fromUri(uriInfo.getRequestUri()); builder.replaceQueryParam("gbrecs", true); + builder.replaceQueryParam("gbrids", gbrids); + crc.setProperty("gbrids", gbrids); String baseUrlEncoded = builder.build().toString(); String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; @@ -719,19 +732,24 @@ public DownloadInstance downloadAuxiliaryFile(@Context ContainerRequestContext c @Path("datafiles") @Consumes("text/plain") @Produces({ "application/zip" }) - public Response postDownloadDatafiles(@Context ContainerRequestContext crc, String body, @QueryParam("gbrecs") boolean gbrecs, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + public Response postDownloadDatafiles(@Context ContainerRequestContext crc, String body, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { processDatafileWithGuestbookResponse(crc, headers, body, uriInfo, gbrecs, body); + // JSF UI passes the guestbook response id(s) in thus this qp can be removed when JSF is removed + if (gbrids == null || gbrids.isEmpty()) { + gbrids = (String) crc.getProperty("gbrids"); + } // There is no get for this so we shouldn't return a signed url // initiate the download now - return downloadDatafiles(crc, body, gbrecs, uriInfo, headers, response, null); + return downloadDatafiles(crc, body, gbrecs, gbrids, uriInfo, headers, response, null); } @GET @AuthRequired @Path("dataset/{id}") @Produces({"application/zip"}) - public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, + public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, + @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { try { User user = getRequestUser(crc); @@ -746,7 +764,7 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat // We don't want downloads from Draft versions to be counted, // so we are setting the gbrecs (aka "do not write guestbook response") // variable accordingly: - return downloadDatafiles(crc, fileIds, true, uriInfo, headers, response, "draft"); + return downloadDatafiles(crc, fileIds, true, gbrids, uriInfo, headers, response, "draft"); } } @@ -767,7 +785,7 @@ public Response downloadAllFromLatest(@Context ContainerRequestContext crc, @Pat } String fileIds = getFileIdsAsCommaSeparated(latest.getFileMetadatas()); - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, latest.getFriendlyVersionNumber()); + return downloadDatafiles(crc, fileIds, gbrecs, gbrids, uriInfo, headers, response, latest.getFriendlyVersionNumber()); } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -807,7 +825,8 @@ public Response downloadAllFromLatestWithGuestbookResponse(@Context ContainerReq @AuthRequired @Path("dataset/{id}/versions/{versionId}") @Produces({"application/zip"}) - public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @PathParam("versionId") String versionId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { + public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetIdOrPersistentId, @PathParam("versionId") String versionId, + @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @QueryParam("key") String apiTokenParam, @QueryParam("signed") boolean signed, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { try { DatasetVersion dsv = getDatasetVersionFromVersion(crc, datasetIdOrPersistentId, versionId); if (dsv == null) { @@ -825,7 +844,7 @@ public Response downloadAllFromVersion(@Context ContainerRequestContext crc, @Pa gbrecs = true; } - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, dsv.getFriendlyVersionNumber().toLowerCase()); + return downloadDatafiles(crc, fileIds, gbrecs, gbrids, uriInfo, headers, response, dsv.getFriendlyVersionNumber().toLowerCase()); } catch (WrappedResponse wr) { return wr.getResponse(); } @@ -908,10 +927,10 @@ private String generateMultiFileBundleName(Dataset dataset, String versionTag) { @AuthRequired @Path("datafiles/{fileIds}") @Produces({"application/zip"}) - public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, + public Response datafiles(@Context ContainerRequestContext crc, @PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException { - return downloadDatafiles(crc, fileIds, gbrecs, uriInfo, headers, response, null); + return downloadDatafiles(crc, fileIds, gbrecs, gbrids, uriInfo, headers, response, null); } @POST @@ -949,7 +968,7 @@ private String[] getFileIdsCSV(String body) { } } - private Response downloadDatafiles(ContainerRequestContext crc, String body, boolean donotwriteGBResponse, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response, String versionTag) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { + private Response downloadDatafiles(ContainerRequestContext crc, String body, boolean donotwriteGBResponse, String gbrids, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response, String versionTag) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { final long zipDownloadSizeLimit = systemConfig.getZipDownloadLimit(); logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes."); @@ -982,8 +1001,13 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo // Get DataFiles, check authorized access, check for multiple Datasets, and check for required guestbook response Set datasetIds = new HashSet<>(); + Boolean guestbookResponseRequired = null; for (int i = 0; i < fileIdParams.length; i++) { DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); + if (guestbookResponseRequired == null) { + // Only need to check this on the first file + guestbookResponseRequired = checkGuestbookRequiredResponse(crc, df, gbrids); + } datafilesMap.put(df.getId(), df); datasetIds.add(df.getOwner() != null ? df.getOwner().getId() : 0L); if (isAccessAuthorized(user, df)) { @@ -992,7 +1016,7 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo if (datasetIds.size() > 1) { // All files must be from the same Dataset return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.multipleDatasets")); - } else if (authorizedDatafileIds.contains(df.getId()) && checkGuestbookRequiredResponse(crc, df)) { + } else if (authorizedDatafileIds.contains(df.getId()) && guestbookResponseRequired) { try { GuestbookResponse gbr = getGuestbookResponseFromBody(df, GuestbookResponse.DOWNLOAD, body, user); if (gbr != null) { @@ -1919,13 +1943,31 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @ return ok(jsonObjectBuilder); } - private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df) { + private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df, String gbrids) throws WebApplicationException { // Check if guestbook response is required - // check if this was originally a signed url - since the property only exists if true then a null check should suffice - boolean notSigned = (crc.getProperty("wasSigned") == null); - boolean wasWrittenInPost = true; // TODO: not required if signed and has some flag stating that the guestbook response was already written + boolean required = df.getOwner().hasEnabledGuestbook(); + boolean wasWrittenInPost = false; + if (required) { + User requestor = getRequestor(crc); + if (requestor instanceof AuthenticatedUser && permissionService.userOn(requestor, df.getOwner()).has(Permission.EditDataset)) { + required = false; + } - return notSigned && !wasWrittenInPost && df.getOwner().hasEnabledGuestbook(); + if (required && gbrids != null && !gbrids.isEmpty()) { + try { + // verify that this id is good + GuestbookResponse gbr = guestbookResponseService.findById(Long.valueOf(gbrids)); + if (gbr == null) { + throw new NotFoundException("GuestbookResponse Not Found for id:" + gbrids); + } + Long delta = Instant.now().toEpochMilli() - gbr.getResponseTime().getTime(); + wasWrittenInPost = gbr.getDataset().getId().equals(df.getOwner().getId()) && delta < 10000; + } catch (NumberFormatException | DateTimeParseException ex) { + throw new BadRequestException(ex.getMessage()); + } + } + } + return required && !wasWrittenInPost; } private GuestbookResponse getGuestbookResponseFromBody(DataFile dataFile, String type, String jsonBody, User requestor) throws JsonParseException { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java index 258770da0d4..d21d91a07c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java @@ -94,7 +94,6 @@ private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext container boolean isSignedUrlValid = UrlSignerUtil.isValidUrl(signedUrl, userId, requestMethod, signedUrlSigningKey); if (isSignedUrlValid) { user = targetUser; - containerRequestContext.setProperty("wasSigned", Boolean.TRUE); } } return user; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 2e3bbb5b568..6ac4a3225fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -23,15 +23,10 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.dataaccess.DataAccess; -import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; -import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; +import edu.harvard.iq.dataverse.dataaccess.*; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException; - -import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT; -import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable; import edu.harvard.iq.dataverse.ingest.IngestReport; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.ingest.IngestableDataChecker; @@ -41,50 +36,9 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.file.BagItFileHandler; import edu.harvard.iq.dataverse.util.file.BagItFileHandlerFactory; +import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException; import edu.harvard.iq.dataverse.util.xml.XmlUtil; import edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatDoc; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.HTML_H1; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.HTML_TABLE_HDR; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatTitle; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatTable; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatTableCell; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatLink; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatTableCellAlignRight; -import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.formatTableRow; - -import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.sql.Timestamp; -import java.text.MessageFormat; -import java.text.SimpleDateFormat; -import java.time.LocalDate; -import java.util.Map; -import java.util.MissingResourceException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.logging.Level; -import java.util.logging.Logger; import jakarta.activation.MimetypesFileTypeMap; import jakarta.ejb.EJBException; import jakarta.enterprise.inject.spi.CDI; @@ -94,28 +48,40 @@ import jakarta.faces.validator.ValidatorException; import jakarta.json.JsonArray; import jakarta.json.JsonObject; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tika.Tika; +import org.primefaces.model.file.UploadedFile; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; import javax.imageio.ImageIO; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; - - -import java.util.zip.GZIPInputStream; -import org.apache.commons.io.FilenameUtils; - -import edu.harvard.iq.dataverse.dataaccess.DataAccessOption; -import edu.harvard.iq.dataverse.dataaccess.StorageIO; -import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException; +import java.awt.image.BufferedImage; +import java.io.*; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import org.apache.commons.lang3.StringUtils; -import org.apache.tika.Tika; -import org.primefaces.model.file.UploadedFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Timestamp; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; -import ucar.nc2.NetcdfFile; -import ucar.nc2.NetcdfFiles; +import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT; +import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable; +import static edu.harvard.iq.dataverse.util.xml.html.HtmlFormatUtil.*; /** * a 4.0 implementation of the DVN FileUtil; @@ -1337,7 +1303,7 @@ public static String getPublicDownloadUrl(String dataverseSiteUrl, String persis /** * The FileDownloadServiceBean operates on file IDs, not DOIs. */ - public static String getFileDownloadUrlPath(String downloadType, Long fileId, boolean gbRecordsWritten, Long fileMetadataId) { + public static String getFileDownloadUrlPath(String downloadType, Long fileId, boolean gbRecordsWritten, Long fileMetadataId, String gbrIds) { String fileDownloadUrl = "/api/access/datafile/" + fileId; if (downloadType != null) { switch(downloadType) { @@ -1371,6 +1337,9 @@ public static String getFileDownloadUrlPath(String downloadType, Long fileId, bo fileDownloadUrl += "?gbrecs=true"; } } + if (gbrIds != null && !gbrIds.isEmpty()) { + fileDownloadUrl += (fileDownloadUrl.contains("?") ? "&" : "?") + "gbrids=" + gbrIds; + } logger.fine("Returning file download url: " + fileDownloadUrl); return fileDownloadUrl; } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index 46359d7b02c..af3ef6694fd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -3,25 +3,20 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.util.FileUtil.FileCitationExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.File; import java.io.IOException; -import java.time.LocalDate; import java.net.URI; +import java.time.LocalDate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Stream; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; public class FileUtilTest { @@ -248,17 +243,17 @@ public void testIsPubliclyDownloadable5() { public void testgetFileDownloadUrl() { Long fileId = 42l; Long fileMetadataId = 2L; - assertEquals("/api/access/datafile/42", FileUtil.getFileDownloadUrlPath(null, fileId, false, null)); - assertEquals("/api/access/datafile/42", FileUtil.getFileDownloadUrlPath("", fileId, false, null)); - assertEquals("/api/access/datafile/bundle/42", FileUtil.getFileDownloadUrlPath("bundle", fileId, false, null)); - assertEquals("/api/access/datafile/bundle/42?fileMetadataId=2", FileUtil.getFileDownloadUrlPath("bundle", fileId, false, fileMetadataId)); - assertEquals("/api/access/datafile/42?format=original", FileUtil.getFileDownloadUrlPath("original", fileId, false, null)); - assertEquals("/api/access/datafile/42?format=RData", FileUtil.getFileDownloadUrlPath("RData", fileId, false, null)); - assertEquals("/api/access/datafile/42/metadata", FileUtil.getFileDownloadUrlPath("var", fileId, false, null)); - assertEquals("/api/access/datafile/42/metadata?fileMetadataId=2", FileUtil.getFileDownloadUrlPath("var", fileId, false, fileMetadataId)); - assertEquals("/api/access/datafile/42?format=tab", FileUtil.getFileDownloadUrlPath("tab", fileId, false, null)); - assertEquals("/api/access/datafile/42?format=tab&gbrecs=true", FileUtil.getFileDownloadUrlPath("tab", fileId, true, null)); - assertEquals("/api/access/datafile/42?gbrecs=true", FileUtil.getFileDownloadUrlPath(null, fileId, true, null)); + assertEquals("/api/access/datafile/42", FileUtil.getFileDownloadUrlPath(null, fileId, false, null, null)); + assertEquals("/api/access/datafile/42", FileUtil.getFileDownloadUrlPath("", fileId, false, null, null)); + assertEquals("/api/access/datafile/bundle/42", FileUtil.getFileDownloadUrlPath("bundle", fileId, false, null, null)); + assertEquals("/api/access/datafile/bundle/42?fileMetadataId=2", FileUtil.getFileDownloadUrlPath("bundle", fileId, false, fileMetadataId, null)); + assertEquals("/api/access/datafile/42?format=original", FileUtil.getFileDownloadUrlPath("original", fileId, false, null, null)); + assertEquals("/api/access/datafile/42?format=RData", FileUtil.getFileDownloadUrlPath("RData", fileId, false, null, null)); + assertEquals("/api/access/datafile/42/metadata", FileUtil.getFileDownloadUrlPath("var", fileId, false, null, null)); + assertEquals("/api/access/datafile/42/metadata?fileMetadataId=2", FileUtil.getFileDownloadUrlPath("var", fileId, false, fileMetadataId, null)); + assertEquals("/api/access/datafile/42?format=tab", FileUtil.getFileDownloadUrlPath("tab", fileId, false, null, null)); + assertEquals("/api/access/datafile/42?format=tab&gbrecs=true", FileUtil.getFileDownloadUrlPath("tab", fileId, true, null, null)); + assertEquals("/api/access/datafile/42?gbrecs=true", FileUtil.getFileDownloadUrlPath(null, fileId, true, null, null)); } @Test From fc0f00324dbfc746b8d37ab0e1b7687557c09030 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 08:59:47 -0400 Subject: [PATCH 30/52] add check for downloading thumbnails --- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index e53b5079f4b..73091b7517d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -236,7 +236,8 @@ private DataFile findDataFileOrDieWrapper(String fileId){ @Produces({"application/xml","*/*"}) public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - + // Check is we are downloading a thumbnail image which doesn't require a guestbook response + boolean imageThumb = uriInfo.getQueryParameters().containsKey("imageThumb") && uriInfo.getQueryParameters().getFirst("imageThumb").equalsIgnoreCase("true"); fileId = normalizeFileId(fileId); DataFile df = findDataFileOrDieWrapper(fileId); @@ -251,7 +252,7 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); User requestor = getRequestor(crc); - if (checkGuestbookRequiredResponse(crc, df, gbrids)) { + if (!imageThumb && checkGuestbookRequiredResponse(crc, df, gbrids)) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } From 726603f8e7d56bb7d3a28b28d783fc2720d64c3f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 09:22:29 -0400 Subject: [PATCH 31/52] add check for downloading thumbnails --- .../edu/harvard/iq/dataverse/api/Access.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 73091b7517d..e3fddd3eca5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -146,7 +146,7 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); User requestor = getRequestor(crc); - if (checkGuestbookRequiredResponse(crc, df, gbrids)) { + if (checkGuestbookRequiredResponse(crc, uriInfo, df, gbrids)) { throw new BadRequestException(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } @@ -236,8 +236,7 @@ private DataFile findDataFileOrDieWrapper(String fileId){ @Produces({"application/xml","*/*"}) public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("gbrids") String gbrids, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ { - // Check is we are downloading a thumbnail image which doesn't require a guestbook response - boolean imageThumb = uriInfo.getQueryParameters().containsKey("imageThumb") && uriInfo.getQueryParameters().getFirst("imageThumb").equalsIgnoreCase("true"); + fileId = normalizeFileId(fileId); DataFile df = findDataFileOrDieWrapper(fileId); @@ -252,7 +251,7 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI // This will throw a ForbiddenException if access isn't authorized: checkAuthorization(crc, df); User requestor = getRequestor(crc); - if (!imageThumb && checkGuestbookRequiredResponse(crc, df, gbrids)) { + if (checkGuestbookRequiredResponse(crc, uriInfo, df, gbrids)) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); } @@ -423,7 +422,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr // since all files must be in the same Dataset we can generate a Guestbook Response once and just replace the DataFile for each file in the list DataFile firstDatafile = datafilesMap.values().size() > 0 ? (DataFile) Arrays.stream(datafilesMap.values().toArray()).findFirst().get() : null; GuestbookResponse gbr = getGuestbookResponseFromBody(firstDatafile, GuestbookResponse.DOWNLOAD, jsonBody, user); - boolean guestbookResponseRequired = checkGuestbookRequiredResponse(crc, firstDatafile, null); + boolean guestbookResponseRequired = checkGuestbookRequiredResponse(crc, uriInfo, firstDatafile, null); for (DataFile df : datafilesMap.values()) { displayName = df.getDisplayName(); if (guestbookResponseRequired) { @@ -1007,7 +1006,7 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo DataFile df = findDataFileOrDieWrapper(fileIdParams[i]); if (guestbookResponseRequired == null) { // Only need to check this on the first file - guestbookResponseRequired = checkGuestbookRequiredResponse(crc, df, gbrids); + guestbookResponseRequired = checkGuestbookRequiredResponse(crc, uriInfo, df, gbrids); } datafilesMap.put(df.getId(), df); datasetIds.add(df.getOwner() != null ? df.getOwner().getId() : 0L); @@ -1944,7 +1943,7 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @ return ok(jsonObjectBuilder); } - private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, DataFile df, String gbrids) throws WebApplicationException { + private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, UriInfo uriInfo, DataFile df, String gbrids) throws WebApplicationException { // Check if guestbook response is required boolean required = df.getOwner().hasEnabledGuestbook(); boolean wasWrittenInPost = false; @@ -1953,6 +1952,11 @@ private boolean checkGuestbookRequiredResponse(ContainerRequestContext crc, Data if (requestor instanceof AuthenticatedUser && permissionService.userOn(requestor, df.getOwner()).has(Permission.EditDataset)) { required = false; } + // Check if we are downloading a thumbnail image which doesn't require a guestbook response + boolean imageThumb = uriInfo.getQueryParameters().containsKey("imageThumb"); + if (imageThumb) { + required = false; + } if (required && gbrids != null && !gbrids.isEmpty()) { try { From 00962b0542f333dd2d744cf1bcf3c43872572972 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:03:49 -0400 Subject: [PATCH 32/52] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ff030352719..26cf43e35a6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1223,7 +1223,7 @@ Get a Guestbook for a Dataverse Collection For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. -Get a Guestbook by it's id +Get a Guestbook by its id .. code-block:: bash From 179d0ea5d0b222bc1d7df93a3f5e92fcaf161222 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:48:33 -0400 Subject: [PATCH 33/52] update docs --- doc/sphinx-guides/source/api/changelog.rst | 12 ++++++++++++ doc/sphinx-guides/source/api/dataaccess.rst | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 4c7a5914b1e..6ba8340beca 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,18 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.10 +----- +- The following GET APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the body using a POST call. + + - **/api/access/datafile/{fileId:.+}** + + - **/api/access/datafiles/{fileIds}** + + - **/api/access/dataset/{id}** + + - **/api/access/dataset/{id}/versions/{versionId}** + v6.9 ---- diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 01431a02345..9e97edc3d3e 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,7 +32,7 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (no version): @@ -61,7 +61,7 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (with version): @@ -95,7 +95,7 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Restricted files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Example :: From 8e63f15eb65b2802a0492609a99c25110759d661 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:53:29 -0400 Subject: [PATCH 34/52] Update src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java Co-authored-by: Philip Durbin --- src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index 8d81b9f57b6..222d265a0c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -27,7 +27,7 @@ public class UrlSignerUtil { public static final String SIGNED_URL_USER="user"; public static final String SIGNED_URL_UNTIL="until"; public static final String SIGNED_URL_KEY="key"; // do not propagate the key since it's a credential - public static final String SIGNED_URL_SIGNED="signed"; // we need to remove this when returning a singed url to prevent a loop of signing + public static final String SIGNED_URL_SIGNED="signed"; // we need to remove this when returning a signed url to prevent a loop of signing public static final List reservedParameters = List.of(SIGNED_URL_UNTIL, SIGNED_URL_USER, SIGNED_URL_METHOD, SIGNED_URL_TOKEN, SIGNED_URL_KEY, SIGNED_URL_SIGNED); /** * From faefe7f5da5f6f0a43575ecbfa7b44e943d98283 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:59:37 -0400 Subject: [PATCH 35/52] update docs --- doc/sphinx-guides/source/api/changelog.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 6ba8340beca..07f67da7d3e 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -9,7 +9,7 @@ This API changelog is experimental and we would love feedback on its usefulness. v6.10 ----- -- The following GET APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the body using a POST call. +- The following GET APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body using a POST call. - **/api/access/datafile/{fileId:.+}** @@ -19,6 +19,12 @@ v6.10 - **/api/access/dataset/{id}/versions/{versionId}** +- The following POST APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body. + + - **/api/access/datafiles** + + - **/api/access/datafile/bundle/{fileId}** + v6.9 ---- From 90fd76d859ef32f255c279e766450c3cd84b0237 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 15:16:55 -0400 Subject: [PATCH 36/52] add created sttaus checks to IT tests --- src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java | 4 ++++ src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 8 +++++--- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index f4cb2cc6b12..9eb9c1d8f1c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -500,6 +500,8 @@ public void testRequestAccess() throws InterruptedException { String pathToJsonFile = "scripts/api/data/dataset-create-new.json"; Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); + createDatasetResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); createDatasetResponse.prettyPrint(); Integer datasetIdNew = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); @@ -586,6 +588,8 @@ public void testRequestAccessWithGuestbook() throws IOException, JsonParseExcept String pathToJsonFile = "scripts/api/data/dataset-create-new.json"; Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); + createDatasetResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); createDatasetResponse.prettyPrint(); Integer datasetIdNew = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); String persistentIdNew = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 26ee900e91b..67c12164155 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3660,6 +3660,8 @@ public void testUploadFilesWithLimits() throws JsonParseException { String apiToken = UtilIT.getApiTokenFromResponse(createUser); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); // Update the dataverse with a datasetFileCountLimit of 1 @@ -3675,10 +3677,10 @@ public void testUploadFilesWithLimits() throws JsonParseException { Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); - Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); - String datasetPersistenceId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); createDatasetResponse.then().assertThat() .statusCode(CREATED.getStatusCode()); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String datasetPersistenceId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); // ------------------------- // Add initial file @@ -3875,6 +3877,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars msgt("testDownloadFileWithGuestbookResponse"); // Create superuser Response createUserResponse = UtilIT.createRandomUser(); + assertEquals(200, createUserResponse.getStatusCode()); String ownerApiToken = UtilIT.getApiTokenFromResponse(createUserResponse); String superusername = UtilIT.getUsernameFromResponse(createUserResponse); UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200); @@ -3994,7 +3997,6 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars String signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); // Download the file using the signed url Response signedUrlResponse = get(signedUrl); - signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); // Get Download Url attempt - Guestbook Response is required but not found diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 5577daaed27..5051f8677d1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -5433,6 +5433,8 @@ public static Guestbook createRandomGuestbook(String ownerAlias, String persiste jsonParsor.parseGuestbook(jsonObj, gb); Response createGuestbookResponse = UtilIT.createGuestbook(ownerAlias, guestbookAsJson, apiToken); + createGuestbookResponse.then().assertThat() + .statusCode(CREATED.getStatusCode()); createGuestbookResponse.prettyPrint(); JsonPath createdGuestbook = JsonPath.from(createGuestbookResponse.body().asString()); Long guestbookId = Long.parseLong(createdGuestbook.getString("data.message").split(" ")[1]); From efef793d4abf4953bc9050e31c4dd2f4662df200 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 15:30:28 -0400 Subject: [PATCH 37/52] create guestbook returns ok --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 5051f8677d1..0927cb5ef71 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -5434,7 +5434,7 @@ public static Guestbook createRandomGuestbook(String ownerAlias, String persiste Response createGuestbookResponse = UtilIT.createGuestbook(ownerAlias, guestbookAsJson, apiToken); createGuestbookResponse.then().assertThat() - .statusCode(CREATED.getStatusCode()); + .statusCode(OK.getStatusCode()); createGuestbookResponse.prettyPrint(); JsonPath createdGuestbook = JsonPath.from(createGuestbookResponse.body().asString()); Long guestbookId = Long.parseLong(createdGuestbook.getString("data.message").split(" ")[1]); From f320a39fbdbd5af26d319d33b1ca2218f83ea274 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:17:02 -0400 Subject: [PATCH 38/52] update docs --- doc/sphinx-guides/source/api/dataaccess.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 9e97edc3d3e..cf5d41cf0fb 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,7 +32,7 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset who's file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (no version): @@ -61,7 +61,7 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset who's file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (with version): @@ -95,7 +95,7 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. Example :: From 135542fee670c3e0439697d872b81f2a6c74eadb Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:27:39 -0400 Subject: [PATCH 39/52] remove prettyPrints --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 67c12164155..d426e6f9e51 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -4016,7 +4016,6 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars // Download the file using the signed url signedUrlResponse = get(signedUrl); - signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); // Download multiple files - Guestbook Response is required but not found for file2 and file3 @@ -4030,14 +4029,11 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars // Download multiple files with guestbook response and fileIds in json String jsonBody = "{\"fileIds\":[" + fileId1 + "," + fileId2+ "," + fileId3 +"], " + guestbookResponse.substring(1); downloadResponse = UtilIT.postDownloadDatafiles(jsonBody, apiToken); - downloadResponse.prettyPrint(); assertEquals(OK.getStatusCode(), downloadResponse.getStatusCode()); downloadResponse = UtilIT.downloadFilesUrlWithGuestbookResponse(new Integer[]{fileId1, fileId2, fileId3}, apiToken, guestbookResponse); - downloadResponse.prettyPrint(); signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); signedUrlResponse = get(signedUrl); - signedUrlResponse.prettyPrint(); assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); // TEST Overwrite name, email, institution and position in guestbook Response. Using user2 @@ -4049,7 +4045,6 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars guestbookResponse = guestbookResponse.replace("\"guestbookResponse\": {", "\"guestbookResponse\": { \"name\":\"My Name\", \"position\":\"My Position\", \"institution\":\"My Institution\","); downloadResponse = UtilIT.getDownloadFileUrlWithGuestbookResponse(fileId1, apiToken2, guestbookResponse); - downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .statusCode(OK.getStatusCode()); Response guestbookResponses = UtilIT.getGuestbookResponses(dataverseAlias, guestbook.getId(), ownerApiToken); From 42f9fcf80172eca054a679b023e3a2e0e0de63e8 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:30:33 -0400 Subject: [PATCH 40/52] remove print statements --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 0927cb5ef71..8def32f7d6a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2164,8 +2164,6 @@ static Response requestFileAccess(String fileIdOrPersistentId, String apiToken) return requestFileAccess(fileIdOrPersistentId, apiToken, null); } static Response requestFileAccess(String fileIdOrPersistentId, String apiToken, String body) { - System.out.print ("Request file access + fileIdOrPersistentId: " + fileIdOrPersistentId); - System.out.print ("Request file access + apiToken: " + apiToken); String idInPath = fileIdOrPersistentId; // Assume it's a number. String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (!NumberUtils.isCreatable(fileIdOrPersistentId)) { From 162a40f3abe12c69ad262d895f614bb6f2c12e65 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 10 Mar 2026 14:51:07 -0400 Subject: [PATCH 41/52] add fix to access request test --- .../java/edu/harvard/iq/dataverse/api/AccessIT.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index 9eb9c1d8f1c..0c229bf28ad 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -28,6 +28,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import static io.restassured.RestAssured.get; import static jakarta.ws.rs.core.Response.Status.*; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; @@ -678,9 +679,14 @@ public void testRequestAccessWithGuestbook() throws IOException, JsonParseExcept requestFileAccessResponse = UtilIT.requestFileAccess(basicFileIdNew.toString(), apiTokenRando); assertEquals(400, requestFileAccessResponse.getStatusCode()); - //Now should be able to download + //Now should be able to download but the guestbook response is still required randoDownload = UtilIT.downloadFile(tabFile3IdRestrictedNew, apiTokenRando); - assertEquals(OK.getStatusCode(), randoDownload.getStatusCode()); + assertEquals(BAD_REQUEST.getStatusCode(), randoDownload.getStatusCode()); + randoDownload = UtilIT.getDownloadFileUrlWithGuestbookResponse(tabFile3IdRestrictedNew, apiTokenRando, guestbookResponseJson); + String signedUrl = UtilIT.getSignedUrlFromResponse(randoDownload); + // Download the file using the signed url + Response signedUrlResponse = get(signedUrl); + assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); //revokeFileAccess Response revokeFileAccessResponse = UtilIT.revokeFileAccess(tabFile3IdRestrictedNew.toString(), "@" + apiIdentifierRando, apiToken); From 1addebdff8e864f9e4f1d2ff2a5eae296e10b7f4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:21:21 -0400 Subject: [PATCH 42/52] adding check for missing 'answers' block in guestbook response --- .../iq/dataverse/util/json/JsonParser.java | 54 ++++++++++--------- .../dataverse/util/json/JsonParserTest.java | 9 ++++ 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 7de50ae0d0b..69b1aa592c6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -635,35 +635,37 @@ public GuestbookResponse parseGuestbookResponse(JsonObject obj, GuestbookRespons Map cqMap = new HashMap<>(); guestbook.getCustomQuestions().stream().forEach(cq -> cqMap.put(cq.getId(),cq)); - JsonArray answers = obj.getJsonArray("answers"); List customQuestionResponses = new ArrayList<>(); - for (JsonObject answer : answers.getValuesAs(JsonObject.class)) { - Long cqId = Long.valueOf(answer.getInt("id")); - // find the matching CustomQuestion - CustomQuestion cq = cqMap.get(cqId); - CustomQuestionResponse cqr = new CustomQuestionResponse(); - cqr.setGuestbookResponse(guestbookResponse); - cqr.setCustomQuestion(cq); - String response = null; - if (cq == null) { - throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound",List.of(cqId.toString()))); - } else if (cq.getQuestionType().equalsIgnoreCase("textarea")) { - String lineFeed = String.valueOf((char) 10); - JsonArray jsonArray = answer.getJsonArray("value"); - List lines = jsonArray.getValuesAs(JsonString.class); - response = lines.stream().map(JsonString::getString).collect(Collectors.joining(lineFeed)); - } else if (cq.getQuestionType().equalsIgnoreCase("options")) { - String option = answer.getString("value"); - if (!cq.getCustomQuestionOptions().contains(option)) { - throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseInvalidOption", List.of(option))); + JsonArray answers = obj.getJsonArray("answers"); + if (answers != null) { + for (JsonObject answer : answers.getValuesAs(JsonObject.class)) { + Long cqId = Long.valueOf(answer.getInt("id")); + // find the matching CustomQuestion + CustomQuestion cq = cqMap.get(cqId); + CustomQuestionResponse cqr = new CustomQuestionResponse(); + cqr.setGuestbookResponse(guestbookResponse); + cqr.setCustomQuestion(cq); + String response = null; + if (cq == null) { + throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound", List.of(cqId.toString()))); + } else if (cq.getQuestionType().equalsIgnoreCase("textarea")) { + String lineFeed = String.valueOf((char) 10); + JsonArray jsonArray = answer.getJsonArray("value"); + List lines = jsonArray.getValuesAs(JsonString.class); + response = lines.stream().map(JsonString::getString).collect(Collectors.joining(lineFeed)); + } else if (cq.getQuestionType().equalsIgnoreCase("options")) { + String option = answer.getString("value"); + if (!cq.getCustomQuestionOptions().contains(option)) { + throw new JsonParseException(BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookresponseInvalidOption", List.of(option))); + } + response = option; + } else { + response = answer.getString("value"); } - response = option; - } else { - response = answer.getString("value"); + cqr.setResponse(response); + customQuestionResponses.add(cqr); + cqMap.remove(cqId); // remove so we can check the remaining for missing required questions } - cqr.setResponse(response); - customQuestionResponses.add(cqr); - cqMap.remove(cqId); // remove so we can check the remaining for missing required questions } guestbookResponse.setCustomQuestionResponses(customQuestionResponses); // verify each required question is in the response diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index e1bfbefe8d5..ba14f5d8d36 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -900,5 +900,14 @@ public void testGuestbookResponse() throws JsonParseException { System.out.println(e.getMessage()); assertTrue(e.getMessage().contains("ID 4 not found")); } + + // Test missing "answers" array + try { + jsonObj = JsonUtil.getJsonObject(guestbookResponseJson.replace("answers", "answer")); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + } catch (JsonParseException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("Guestbook Response entry is required but not present")); + } } } From f6d5e8d55fc74c078235488d95bca8e97d787faa Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:12:18 -0400 Subject: [PATCH 43/52] adding test for access request with gustbook response required but guestbook-at-request not set --- .../java/edu/harvard/iq/dataverse/api/AccessIT.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index 0c229bf28ad..fb6ccdec977 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -497,7 +497,7 @@ private HashMap readZipResponse(InputStream iStrea } @Test - public void testRequestAccess() throws InterruptedException { + public void testRequestAccess() throws InterruptedException, IOException, JsonParseException { String pathToJsonFile = "scripts/api/data/dataset-create-new.json"; Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken); @@ -505,6 +505,13 @@ public void testRequestAccess() throws InterruptedException { .statusCode(CREATED.getStatusCode()); createDatasetResponse.prettyPrint(); Integer datasetIdNew = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + String persistentIdNew = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + + // Test without guestbook-at-request=true the required guestbook response will not prevent the access request from succeeding + // Create a Guestbook + Guestbook guestbook = UtilIT.createRandomGuestbook(dataverseAlias, persistentId, apiToken); + // Set the guestbook on the Dataset + UtilIT.updateDatasetGuestbook(persistentIdNew, guestbook.getId(), apiToken).prettyPrint(); basicFileName = "004.txt"; String basicPathToFile = "scripts/search/data/replace_test/" + basicFileName; @@ -570,7 +577,8 @@ public void testRequestAccess() throws InterruptedException { requestFileAccessResponse = UtilIT.requestFileAccess(basicFileIdNew.toString(), apiTokenRando); assertEquals(400, requestFileAccessResponse.getStatusCode()); - + // disable the guestbook so we can download without guestbook response + UtilIT.enableGuestbook(dataverseAlias, guestbook.getId(), apiToken, "false").prettyPrint(); //Now should be able to download randoDownload = UtilIT.downloadFile(tabFile3IdRestrictedNew, apiTokenRando); assertEquals(OK.getStatusCode(), randoDownload.getStatusCode()); From ac3fd32f2a04746f5f0f281df32b83b0226ece6f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:15:36 -0400 Subject: [PATCH 44/52] fix signedurl with persistentId to replace it database id --- .../edu/harvard/iq/dataverse/api/Access.java | 8 ++- .../edu/harvard/iq/dataverse/api/FilesIT.java | 50 ++------------ .../edu/harvard/iq/dataverse/api/UtilIT.java | 66 +++++++++---------- 3 files changed, 45 insertions(+), 79 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index e3fddd3eca5..490c54fe7ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -418,6 +418,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr // Handle Guestbook Responses String displayName = ""; String gbrids = ""; + Long datasetId = null; try { // since all files must be in the same Dataset we can generate a Guestbook Response once and just replace the DataFile for each file in the list DataFile firstDatafile = datafilesMap.values().size() > 0 ? (DataFile) Arrays.stream(datafilesMap.values().toArray()).findFirst().get() : null; @@ -425,6 +426,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr boolean guestbookResponseRequired = checkGuestbookRequiredResponse(crc, uriInfo, firstDatafile, null); for (DataFile df : datafilesMap.values()) { displayName = df.getDisplayName(); + datasetId = df.getOwner().getId(); if (guestbookResponseRequired) { if (gbr != null) { gbr.setDataFile(df); @@ -447,7 +449,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr List args = Arrays.asList(displayName, ex.getLocalizedMessage()); return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbook.commandError", args)); } - return returnSignedUrl(crc, uriInfo, user, gbrids); + return returnSignedUrl(crc, uriInfo, user, datasetId.toString(), gbrids); } private Map getDatafilesMap(ContainerRequestContext crc, String fileIds) { @@ -473,7 +475,7 @@ private Map getDatafilesMap(ContainerRequestContext crc, String return datafilesMap; } - private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, User user, String gbrids) { + private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, User user, String id, String gbrids) { // Create the signed URL String userIdentifier = null; String key = null; @@ -493,9 +495,11 @@ private Response returnSignedUrl(ContainerRequestContext crc, UriInfo uriInfo, U UriBuilder builder = UriBuilder.fromUri(uriInfo.getRequestUri()); builder.replaceQueryParam("gbrecs", true); builder.replaceQueryParam("gbrids", gbrids); + builder.replaceQueryParam("persistentId", null); // remove this as a parm and add the id to the path crc.setProperty("gbrids", gbrids); String baseUrlEncoded = builder.build().toString(); String baseUrl = URLDecoder.decode(baseUrlEncoded, StandardCharsets.UTF_8); + baseUrl = baseUrl.replace(":persistentId", id); key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; String signedUrl = UrlSignerUtil.signUrl(baseUrl, 1, userIdentifier, "GET", key); return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl)); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index d426e6f9e51..4ebd97bdc59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -4049,53 +4049,15 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars .statusCode(OK.getStatusCode()); Response guestbookResponses = UtilIT.getGuestbookResponses(dataverseAlias, guestbook.getId(), ownerApiToken); assertTrue(guestbookResponses.prettyPrint().contains("My Name," + user2Email + ",My Institution,My Position")); - } - - @Test - public void testDownloadFileWithSignedUrl() throws IOException, JsonParseException { - msgt("testDownloadFileWithSignedUrl"); - // Create superuser - Response createUserResponse = UtilIT.createRandomUser(); - String ownerApiToken = UtilIT.getApiTokenFromResponse(createUserResponse); - String superusername = UtilIT.getUsernameFromResponse(createUserResponse); - UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200); - - // Create Dataverse - String dataverseAlias = createDataverseGetAlias(ownerApiToken); - // Create user with no permission - createUserResponse = UtilIT.createRandomUser(); - assertEquals(200, createUserResponse.getStatusCode()); - String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); - String username = UtilIT.getUsernameFromResponse(createUserResponse); - - // Create Dataset - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, ownerApiToken); - createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); - Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); - String persistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); - Response getDatasetMetadata = UtilIT.nativeGet(datasetId, ownerApiToken); - getDatasetMetadata.then().assertThat().statusCode(200); - - // Upload files - JsonObjectBuilder json1 = Json.createObjectBuilder().add("description", "my description1").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); - Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/dataverseproject.png", json1.build(), ownerApiToken); - uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); - Integer fileId1 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); - JsonObjectBuilder json2 = Json.createObjectBuilder().add("description", "my description2").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); - uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/orcid_16x16.png", json1.build(), ownerApiToken); - uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); - Integer fileId2 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); - JsonObjectBuilder json3 = Json.createObjectBuilder().add("description", "my description3").add("directoryLabel", "data/subdir1").add("categories", Json.createArrayBuilder().add("Data")); - uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), "src/main/webapp/resources/images/cc0.png", json1.build(), ownerApiToken); - uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); - Integer fileId3 = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); - - // Get Signed Download Url with guestbook response - // downloadFile(Integer fileId, String byteRange, String format, String imageThumb, String apiToken) - Response downloadResponse = UtilIT.downloadFile(fileId1, "&signed=true&format=original" , ownerApiToken); + // Get Signed Download Url with guestbook response using persistentId + // POST /api/access/dataset/:persistentId?persistentId=doi:10.xxxx/FK2/ABC + downloadResponse = UtilIT.downloadFilesUrlWithGuestbookResponse(persistentId, apiToken, guestbookResponse); downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .statusCode(OK.getStatusCode()); + signedUrl = UtilIT.getSignedUrlFromResponse(downloadResponse); + signedUrlResponse = get(signedUrl); + assertEquals(OK.getStatusCode(), signedUrlResponse.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 918d7416d08..b5f87f31c27 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1,65 +1,54 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.*; -import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; -import io.restassured.http.ContentType; -import io.restassured.path.json.JsonPath; -import io.restassured.response.Response; - -import java.io.*; -import java.util.*; -import java.util.logging.Logger; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonObject; - -import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; -import static jakarta.ws.rs.core.HttpHeaders.ACCEPT_LANGUAGE; -import static jakarta.ws.rs.core.Response.Status.CREATED; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.util.logging.Level; -import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; -import io.restassured.path.xml.XmlPath; -import edu.harvard.iq.dataverse.mydata.MyDataFilterParams; -import jakarta.ws.rs.core.HttpHeaders; -import org.apache.commons.lang3.StringUtils; -import org.assertj.core.util.Lists; -import org.junit.jupiter.api.Test; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import io.restassured.specification.RequestSpecification; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import com.mashape.unirest.request.GetRequest; import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; +import edu.harvard.iq.dataverse.mydata.MyDataFilterParams; import edu.harvard.iq.dataverse.settings.FeatureFlags; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; +import io.restassured.http.ContentType; +import io.restassured.path.json.JsonPath; +import io.restassured.path.xml.XmlPath; +import io.restassured.response.Response; +import io.restassured.specification.RequestSpecification; import jakarta.json.*; +import jakarta.ws.rs.core.HttpHeaders; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.assertj.core.util.Lists; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; import static edu.harvard.iq.dataverse.api.ApiConstants.*; +import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import static io.restassured.RestAssured.given; import static io.restassured.path.xml.XmlPath.from; +import static jakarta.ws.rs.core.HttpHeaders.ACCEPT_LANGUAGE; +import static jakarta.ws.rs.core.Response.Status.CREATED; import static jakarta.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.startsWith; import static org.junit.jupiter.api.Assertions.*; @@ -1296,6 +1285,17 @@ static Response downloadFilesUrlWithGuestbookResponse(Integer[] fileIds, String } return requestSpecification.post(getString); } + static Response downloadFilesUrlWithGuestbookResponse(String persistentId, String apiToken, String body) { + RequestSpecification requestSpecification = given(); + if (apiToken != null) { + requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken); + } + if (body != null) { + requestSpecification.body(body); + } + String getString = "/api/access/dataset/:persistentId?persistentId=" + persistentId; + return requestSpecification.post(getString); + } static Response postDownloadDatafiles(String body, String apiToken) { String getString = "/api/access/datafiles"; From 8e8966d1eef476bda068270b8c250ba4f46dff6c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:55:37 -0400 Subject: [PATCH 45/52] another empty answers unit test --- .../harvard/iq/dataverse/util/json/JsonParserTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index ba14f5d8d36..cc78dfbc97c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -909,5 +909,13 @@ public void testGuestbookResponse() throws JsonParseException { System.out.println(e.getMessage()); assertTrue(e.getMessage().contains("Guestbook Response entry is required but not present")); } + // Test missing "answers" empty array + try { + jsonObj = JsonUtil.getJsonObject("{\"answers\" : []}"); + gbr = sut.parseGuestbookResponse(jsonObj, guestbookResponse); + } catch (JsonParseException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("Guestbook Response entry is required but not present")); + } } } From 30be73c54a8dc8930156a5532aeab078c228b918 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 12 Mar 2026 16:55:54 -0400 Subject: [PATCH 46/52] typo --- doc/sphinx-guides/source/api/dataaccess.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index cf5d41cf0fb..fe239e74096 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,7 +32,7 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset who's file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (no version): @@ -61,7 +61,7 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset who's file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (with version): From 149ebde6a574882be074e283a5ecd73b19e2265c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Mar 2026 09:51:23 -0400 Subject: [PATCH 47/52] fix typo guestbook_id to guestbookId in json --- doc/sphinx-guides/source/api/dataaccess.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index fe239e74096..34f6aa4f7e4 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,7 +32,7 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (no version): @@ -61,7 +61,7 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (with version): @@ -95,7 +95,7 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbook_id from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. Example :: From 05a961ed645b0fe7f954fddf6defa0504ebdb984 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Mar 2026 10:14:25 -0400 Subject: [PATCH 48/52] adding to docs --- doc/sphinx-guides/source/api/dataaccess.rst | 6 +++--- doc/sphinx-guides/source/api/native-api.rst | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 34f6aa4f7e4..5d369990c49 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -32,7 +32,7 @@ Basic Download By Dataset The basic form downloads files from the latest accessible version of the dataset. If you are not using an API token, this means the most recently published version. If you are using an API token with full access to the dataset, this means the draft version or the most recently published version if no draft exists. -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset (See :ref:`dataset-json-representation` to get the Dataset with guestbookId) to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (no version): @@ -61,7 +61,7 @@ The second form of the "download by dataset" API allows you to specify which ver * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. To determine what information is required in the response you must first get the Guestbook for the Dataset whose file(s) you are trying to access along with any Custom Questions. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset (See :ref:`dataset-json-representation` to get the Dataset with guestbookId) to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. A curl example using a DOI (with version): @@ -95,7 +95,7 @@ Basic access URI: GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB -.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. +.. note:: Files that require a Guestbook Response will require an additional step to supply the Guestbook Response. A POST to the same endpoint with the Guestbook Response in the body can return a signed url that can be used to download the file(s) via a browser or download manager. For more about guestbooks, see :ref:`dataset-guestbooks` in the User Guide. In the following JSON example please note that the `name`, `email`, `institution`, and `position` fields will default to the User's account information if not included in the response. Call ``GET http://$SERVER/api/guestbooks/{id}`` with the guestbookId from the Dataset (See :ref:`dataset-json-representation` to get the Dataset with guestbookId) to retrieve the Guestbook and Custom Questions. Build the JSON response with the information requested and add it to the body of the POST call within "guestbookResponse":{}. Example :: diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 0f65d3b0fe2..808dbeec815 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1868,6 +1868,8 @@ In all commands below, dataset versions can be referred to as: * ``x.y`` a specific version, where ``x`` is the major version number and ``y`` is the minor version number. * ``x`` same as ``x.0`` +.. _dataset-json-representation: + Get JSON Representation of a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 879c4655c8060decf9588eda4c380483b4c7eecb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 13 Mar 2026 10:33:11 -0400 Subject: [PATCH 49/52] give downloaders more hints to find the guestbookId #12001 --- doc/sphinx-guides/source/api/changelog.rst | 4 ++-- doc/sphinx-guides/source/api/dataaccess.rst | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 7e09410e5ad..d9154ebc838 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -9,7 +9,7 @@ This API changelog is experimental and we would love feedback on its usefulness. v6.10 ----- -- The following GET APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body using a POST call. +- The following GET APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body using a POST call. See the notes under :ref:`basic-file-access` and :ref:`download-by-dataset-by-version` for details. - **/api/access/datafile/{fileId:.+}** @@ -19,7 +19,7 @@ v6.10 - **/api/access/dataset/{id}/versions/{versionId}** -- The following POST APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body. +- The following POST APIs will now return ``400`` if a required Guestbook Response is not supplied. A Guestbook Response can be passed to these APIs in the JSON body. See the note under :ref:`basic-download-by-dataset` for details. - **/api/access/datafiles** diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 5d369990c49..80d9a4d9ed2 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -27,6 +27,8 @@ Please note that in addition to the files from dataset, an additional file call There are two forms of the "download by dataset" API, a basic form and one that supports dataset versions. +.. _basic-download-by-dataset: + Basic Download By Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -50,6 +52,8 @@ The fully expanded example above (without environment variables) looks like this curl -L -O -J -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/access/dataset/:persistentId/?persistentId=doi:10.70122/FK2/N2XGBJ +.. _download-by-dataset-by-version: + Download By Dataset By Version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -82,6 +86,8 @@ The fully expanded example above (without environment variables) looks like this curl -O -J -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/access/dataset/:persistentId/versions/2.0?persistentId=doi:10.70122/FK2/N2XGBJ +.. _basic-file-access: + Basic File Access ----------------- From 35a1eb238c7b6998bbebc4ddacfa27a4a6a42c5a Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Mar 2026 10:38:30 -0400 Subject: [PATCH 50/52] add guestbookId to missing response message --- .../java/edu/harvard/iq/dataverse/api/Access.java | 13 +++++++++---- src/main/java/propertyFiles/Bundle.properties | 2 +- .../java/edu/harvard/iq/dataverse/api/FilesIT.java | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 490c54fe7ca..823715de7de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -147,7 +147,7 @@ public BundleDownloadInstance datafileBundle(@Context ContainerRequestContext cr checkAuthorization(crc, df); User requestor = getRequestor(crc); if (checkGuestbookRequiredResponse(crc, uriInfo, df, gbrids)) { - throw new BadRequestException(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + throw new BadRequestException(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", getGuestbookIdFromDatafile(df))); } if (gbrecs != true && df.isReleased()) { @@ -252,7 +252,7 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI checkAuthorization(crc, df); User requestor = getRequestor(crc); if (checkGuestbookRequiredResponse(crc, uriInfo, df, gbrids)) { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", getGuestbookIdFromDatafile(df))); } if (gbrecs != true && df.isReleased()){ @@ -407,6 +407,11 @@ private String normalizeFileId(String fileId) { return fId; } + // for bundle arg list + private List getGuestbookIdFromDatafile(DataFile df) { + return df != null && df.getOwner() != null && df.getOwner().getGuestbook() != null ? List.of(df.getOwner().getGuestbook().getId().toString()) : List.of(); + } + // Process the guestbook response from JSON and return a signedUrl to the matching GET call private Response processDatafileWithGuestbookResponse(ContainerRequestContext crc, HttpHeaders headers, String fileIds, UriInfo uriInfo, boolean gbrecs, String jsonBody) { @@ -435,7 +440,7 @@ private Response processDatafileWithGuestbookResponse(ContainerRequestContext cr MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); mdcLogService.logEntry(entry); } else { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", getGuestbookIdFromDatafile(df))); } } else if (gbrecs != true && df.isReleased()) { // Write Guestbook record if not done previously and file is released @@ -1031,7 +1036,7 @@ private Response downloadDatafiles(ContainerRequestContext crc, String body, boo MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, df); mdcLogService.logEntry(entry); } else { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing")); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", getGuestbookIdFromDatafile(df))); } } catch (JsonParseException | CommandException ex) { List args = Arrays.asList(df.getDisplayName(), ex.getLocalizedMessage()); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 99f10e66dd2..933b696d6ce 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2953,7 +2953,7 @@ access.api.exception.metadata.not.available.for.nontabular.file=This type of met access.api.exception.metadata.restricted.no.permission=You do not have permission to download this file. access.api.exception.version.not.found=Could not find requested dataset version. access.api.exception.dataset.not.found=Could not find requested dataset. -access.api.download.failure.guestbookResponseMissing=You may not download this file without the required Guestbook response. +access.api.download.failure.guestbookResponseMissing=You may not download this file without the required Guestbook response for guestbookID {0}. access.api.download.failure.guestbook.commandError=Problem trying download with guestbook response on {0} : {1} access.api.download.failure.multipleDatasets=All files being downloaded must be from the same Dataset. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 4ebd97bdc59..ed96b5b4656 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3978,7 +3978,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", List.of(guestbook.getId().toString())))) .statusCode(BAD_REQUEST.getStatusCode()); // With GuestbookResponse. Guest user doesn't have the required Name and Email. so this will still fail downloadResponse = UtilIT.postDownloadFile(fileId4, guestbookResponse); @@ -4004,7 +4004,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", List.of(guestbook.getId().toString())))) .statusCode(BAD_REQUEST.getStatusCode()); // Get Signed Download Url with guestbook response @@ -4023,7 +4023,7 @@ public void testDownloadFileWithGuestbookResponse() throws IOException, JsonPars downloadResponse.prettyPrint(); downloadResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing"))) + .body("message", equalTo(BundleUtil.getStringFromBundle("access.api.download.failure.guestbookResponseMissing", List.of(guestbook.getId().toString())))) .statusCode(BAD_REQUEST.getStatusCode()); // Download multiple files with guestbook response and fileIds in json From d9b59ca3d55c07bbb457809ddca0a428714911bc Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:55:12 -0400 Subject: [PATCH 51/52] doc change --- doc/sphinx-guides/source/api/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index d9154ebc838..fd2dd68f4c4 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -25,6 +25,11 @@ v6.10 - **/api/access/datafile/bundle/{fileId}** +- The following PUT APIs will now return ``400`` if a required Guestbook Response is not supplied. When JVM setting -Ddataverse.files.guestbook-at-request=true is set a Guestbook Response may be required to be passed to these APIs in the JSON body. See the note under Configuration :ref:`dataverse.files.guestbook-at-request` for details. + + - **/api/access/datafile/{id}/requestAccess** + + v6.9 ---- From 63db9036e7071fd4f318f36a0623e5d1efe2d8a6 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 13 Mar 2026 12:03:39 -0400 Subject: [PATCH 52/52] add guestbookId to request access error message --- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 823715de7de..f7654720b71 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1655,7 +1655,7 @@ public Response requestFileAccess(@Context ContainerRequestContext crc GuestbookResponse guestbookResponse = getGuestbookResponseFromBody(dataFile, GuestbookResponse.ACCESS_REQUEST, jsonBody, getRequestUser(crc)); if (ds.getGuestbook() != null && ds.getGuestbook().isEnabled()) { if (ds.getEffectiveGuestbookEntryAtRequest() && guestbookResponse == null) { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookAccessRequestResponseMissing")); + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.guestbookAccessRequestResponseMissing", List.of(ds.getGuestbook().getId().toString()))); } else if (guestbookResponse != null) { engineSvc.submit(new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset())); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 933b696d6ce..f7506087744 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2928,7 +2928,7 @@ access.api.requestAccess.failure.commandError=Problem trying request access on { access.api.requestAccess.failure.requestExists=An access request for this file on your behalf already exists. access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. -access.api.requestAccess.failure.guestbookAccessRequestResponseMissing=You may not request access to this file without the required Guestbook response. +access.api.requestAccess.failure.guestbookAccessRequestResponseMissing=You may not request access to this file without the required Guestbook response for guestbookID {0}. access.api.requestAccess.failure.guestbookresponseMissingRequired=Guestbook Response entry is required but not present ({0}). access.api.requestAccess.failure.guestbookresponseInvalidOption=Guestbook Custom Question Answer not a valid option ({0}). access.api.requestAccess.failure.guestbookresponseQuestionIdNotFound=Guestbook Custom Question ID {0} not found.