diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java
index 9fe130c..1330f41 100644
--- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java
+++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java
@@ -8,10 +8,19 @@
*/
package org.eclipse.hawkbit.artifact.repository;
+import java.io.IOException;
import java.io.InputStream;
+import java.util.Base64;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.google.common.io.BaseEncoding;
+import org.apache.http.client.methods.HttpRequestBase;
import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.amazonaws.services.s3.AmazonS3;
@@ -20,13 +29,25 @@
* An {@link AbstractDbArtifact} implementation which retrieves the
* {@link InputStream} from the {@link AmazonS3} client.
*/
-public class S3Artifact extends AbstractDbArtifact {
+public final class S3Artifact extends AbstractDbArtifact {
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3Artifact.class);
private final AmazonS3 amazonS3;
private final S3RepositoryProperties s3Properties;
private final String key;
+ private S3Object s3Object;
+ private WrappedS3InputStream s3InputStream;
+
+ private S3Artifact(final S3Object s3Object, final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties,
+ final String key, final String artifactId, final DbArtifactHash hashes, final Long size,
+ final String contentType) {
+ this(amazonS3, s3Properties, key, artifactId, hashes, size, contentType);
+ this.s3Object = s3Object;
+ this.s3InputStream = WrappedS3InputStream.wrap(s3Object.getObjectContent());
+ }
- S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key,
+ private S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key,
final String artifactId, final DbArtifactHash hashes, final Long size, final String contentType) {
super(artifactId, hashes, size, contentType);
Assert.notNull(amazonS3, "S3 cannot be null");
@@ -37,9 +58,62 @@ public class S3Artifact extends AbstractDbArtifact {
this.key = key;
}
- @Override
- public InputStream getFileInputStream() {
- return amazonS3.getObject(s3Properties.getBucketName(), key).getObjectContent();
+ /**
+ * Get an S3Artifact for an already existing binary in the repository based on
+ * the given key.
+ *
+ * @param amazonS3
+ * connection to the AmazonS3
+ * @param s3Properties
+ * used to retrieve the bucket name
+ * @param key
+ * of the artifact
+ * @param artifactId
+ * sha1Hash to create the {@link DbArtifactHash}
+ * @return an instance of {@link S3Artifact}
+ * @throws S3ArtifactNotFoundException
+ * in case that no artifact could be found for the given values
+ */
+ public static S3Artifact get(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key,
+ final String artifactId) {
+ final S3Object s3Object = getS3ObjectOrThrowException(amazonS3, s3Properties.getBucketName(), key);
+
+ final ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
+ final DbArtifactHash artifactHash = createArtifactHash(artifactId, objectMetadata);
+ return new S3Artifact(s3Object, amazonS3, s3Properties, key, artifactId, artifactHash,
+ objectMetadata.getContentLength(), objectMetadata.getContentType());
+ }
+
+ /**
+ * Create a new instance of {@link S3Artifact}. In this case it is not checked
+ * if an artifact with the given values exists. The S3 object is empty.
+ *
+ * @param amazonS3
+ * connection to the AmazonS3
+ * @param s3Properties
+ * used to retrieve the bucket name
+ * @param key
+ * of the artifact
+ * @param hashes
+ * instance of {@link DbArtifactHash}
+ * @param size
+ * of the artifact
+ * @param contentType
+ * of the artifact
+ * @return an instance of {@link S3Artifact} with an empty {@link S3Object}
+ */
+ public static S3Artifact create(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties,
+ final String key, final DbArtifactHash hashes, final Long size, final String contentType) {
+ return new S3Artifact(amazonS3, s3Properties, key, hashes.getSha1(), hashes, size, contentType);
+ }
+
+ /**
+ * Verify if the {@link S3Object} exists
+ *
+ * @return result of {@link AmazonS3}#doesObjectExist
+ */
+ public boolean exists() {
+ return amazonS3.doesObjectExist(s3Properties.getBucketName(), key);
}
@Override
@@ -47,4 +121,69 @@ public String toString() {
return "S3Artifact [key=" + key + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes()
+ ", getSize()=" + getSize() + ", getContentType()=" + getContentType() + "]";
}
+
+ @Override
+ public InputStream getFileInputStream() {
+ LOG.debug("Get file input stream for s3 object with key {}", key);
+ refreshS3ObjectIfNeeded();
+ return s3InputStream;
+ }
+
+ private void refreshS3ObjectIfNeeded() {
+ if (s3Object == null || s3InputStream == null) {
+ LOG.info("Initialize S3Object in bucket {} with key {}", s3Properties.getBucketName(), key);
+ s3Object = amazonS3.getObject(s3Properties.getBucketName(), key);
+ s3InputStream = WrappedS3InputStream.wrap(s3Object.getObjectContent());
+ }
+ }
+
+ private static S3Object getS3ObjectOrThrowException(AmazonS3 amazonS3, String bucketName, String key) {
+ final S3Object s3Object = amazonS3.getObject(bucketName, key);
+ if (s3Object == null) {
+ throw new S3ArtifactNotFoundException("Cannot find s3 object by given arguments.", bucketName, key);
+ }
+ return s3Object;
+ }
+
+ private static DbArtifactHash createArtifactHash(final String artifactId, ObjectMetadata metadata) {
+ return new DbArtifactHash(artifactId, BaseEncoding.base16().lowerCase()
+ .encode(Base64.getDecoder().decode(sanitizeEtag(metadata.getETag()))), null);
+ }
+
+ private static String sanitizeEtag(final String etag) {
+ // base64 alphabet consist of alphanumeric characters and + / = (see RFC
+ // 4648)
+ return etag.trim().replaceAll("[^A-Za-z0-9+/=]", "");
+ }
+
+ /**
+ * Wrapper to abort the http request of the S3 input stream before closing it
+ */
+ static final class WrappedS3InputStream extends S3ObjectInputStream {
+
+ /**
+ * Constructor
+ */
+ private WrappedS3InputStream(InputStream in, HttpRequestBase httpRequest) {
+ super(in, httpRequest);
+ }
+
+ /**
+ * Wrap an input stream of type {@link S3ObjectInputStream} to abort a
+ * connection before closing the stream
+ *
+ * @param inputStream
+ * the {@link S3ObjectInputStream}
+ * @return an instance of {@link WrappedS3InputStream}
+ */
+ public static WrappedS3InputStream wrap(final S3ObjectInputStream inputStream) {
+ return new WrappedS3InputStream(inputStream, inputStream.getHttpRequest());
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.abort();
+ super.close();
+ }
+ }
}
diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java
new file mode 100644
index 0000000..f8cb73c
--- /dev/null
+++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java
@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2021 Bosch.IO GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+/**
+ * An exception that is thrown as soon as an S3 object could not be found in a S3 bucket.
+ */
+public class S3ArtifactNotFoundException extends RuntimeException {
+
+ private final String bucket;
+ private final String key;
+
+ /**
+ * Constructor with individual error message and information about the searched
+ * artifact.
+ *
+ * @param message
+ * use an individual error message here.
+ *
+ * @param bucket
+ * the bucket of the searched artifact.
+ * @param key
+ * the key of the searched artifact (mostly kind of
+ * 'tenant/sha1hash').
+ */
+ public S3ArtifactNotFoundException(final String message, final String bucket, final String key) {
+ super(message);
+ this.bucket = bucket;
+ this.key = key;
+ }
+
+ /**
+ * Constructor with individual error message with a cause and information about
+ * the searched artifact.
+ *
+ * @param message
+ * use an individual error message here.
+ *
+ * @param cause
+ * the cause of the exception.
+ * @param bucket
+ * the bucket of the searched artifact.
+ * @param key
+ * the key of the searched artifact (mostly kind of
+ * 'tenant/sha1hash').
+ */
+ public S3ArtifactNotFoundException(final String message, final Throwable cause, final String bucket,
+ final String key) {
+ super(message, cause);
+ this.bucket = bucket;
+ this.key = key;
+ }
+
+ /**
+ * Constructor with a cause and information about the searched artifact.
+ *
+ * @param cause
+ * the cause of the exception.
+ * @param bucket
+ * the bucket of the searched artifact.
+ * @param key
+ * the key of the searched artifact (mostly kind of
+ * 'tenant/sha1hash').
+ */
+ public S3ArtifactNotFoundException(final Throwable cause, final String bucket, final String key) {
+ super(cause);
+ this.bucket = bucket;
+ this.key = key;
+ }
+
+ /**
+ * @return key (mostly kind of 'tenant/sha1hash').
+ */
+ public String getKey() {
+ return key;
+ }
+
+ /**
+ * @return the bucket name
+ */
+ public String getBucket() {
+ return bucket;
+ }
+}
diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java
index 34bc4d3..1c3549c 100644
--- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java
+++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java
@@ -28,7 +28,6 @@
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.common.io.BaseEncoding;
@@ -77,7 +76,7 @@ protected AbstractDbArtifact store(final String tenant, final DbArtifactHash bas
LOG.info("Storing file {} with length {} to AWS S3 bucket {} with key {}", file.getName(), file.length(),
s3Properties.getBucketName(), key);
- if (existsByTenantAndSha1(tenant, base16Hashes.getSha1())) {
+ if (s3Artifact.exists()) {
LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", key,
s3Properties.getBucketName());
return s3Artifact;
@@ -100,8 +99,8 @@ protected AbstractDbArtifact store(final String tenant, final DbArtifactHash bas
private S3Artifact createS3Artifact(final String tenant, final DbArtifactHash hashes, final String contentType,
final File file) {
- return new S3Artifact(amazonS3, s3Properties, objectKey(tenant, hashes.getSha1()), hashes.getSha1(), hashes,
- file.length(), contentType);
+ return S3Artifact.create(amazonS3, s3Properties, objectKey(tenant, hashes.getSha1()), hashes, file.length(),
+ contentType);
}
private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType, final File file) {
@@ -131,37 +130,22 @@ private static String objectKey(final String tenant, final String sha1Hash) {
@Override
public AbstractDbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) {
final String key = objectKey(tenant, sha1Hash);
-
- LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
- try (final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), key)) {
- if (s3Object == null) {
- return null;
- }
-
- final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata();
-
- // the MD5Content is stored in the ETag
- return new S3Artifact(amazonS3, s3Properties, key, sha1Hash,
- new DbArtifactHash(sha1Hash,
- BaseEncoding.base16().lowerCase().encode(
- BaseEncoding.base64().decode(sanitizeEtag(s3ObjectMetadata.getETag()))),
- null),
- s3ObjectMetadata.getContentLength(), s3ObjectMetadata.getContentType());
- } catch (final IOException e) {
- LOG.error("Could not verify S3Object", e);
+ LOG.debug("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
+ try {
+ return S3Artifact.get(amazonS3, s3Properties, key, sha1Hash);
+ } catch (final S3ArtifactNotFoundException e) {
+ LOG.debug("Cannot find artifact for bucket {} with key {}", e.getBucket(), e.getKey(), e);
return null;
}
}
- private static String sanitizeEtag(final String etag) {
- // base64 alphabet consist of alphanumeric characters and + / = (see RFC
- // 4648)
- return etag.trim().replaceAll("[^A-Za-z0-9+/=]", "");
- }
-
@Override
public boolean existsByTenantAndSha1(final String tenant, final String sha1Hash) {
- return amazonS3.doesObjectExist(s3Properties.getBucketName(), objectKey(tenant, sha1Hash));
+ final boolean exists = amazonS3.doesObjectExist(s3Properties.getBucketName(), objectKey(tenant, sha1Hash));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Search for artifact with sha1Hash {} results in status: {}", sha1Hash, exists);
+ }
+ return exists;
}
@Override
diff --git a/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java b/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java
index 87d4d00..7a1300c 100644
--- a/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java
+++ b/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java
@@ -28,6 +28,8 @@
import java.security.NoSuchAlgorithmException;
import java.util.Random;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import org.apache.http.client.methods.HttpRequestBase;
import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
import org.junit.jupiter.api.Test;
@@ -70,6 +72,9 @@ public class S3RepositoryTest {
@Mock
private ObjectMetadata s3ObjectMetadataMock;
+ @Mock
+ private S3ObjectInputStream s3ObjectInputStream;
+
@Mock
private PutObjectResult putObjectResultMock;
@@ -121,6 +126,7 @@ public void getArtifactBySHA1Hash() {
final String knownMdBase16 = BaseEncoding.base16().lowerCase().encode(knownMd5.getBytes());
final String knownMd5Base64 = BaseEncoding.base64().encode(knownMd5.getBytes());
+ when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream);
when(amazonS3Mock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock);
when(s3ObjectMock.getObjectMetadata()).thenReturn(s3ObjectMetadataMock);
when(s3ObjectMetadataMock.getContentLength()).thenReturn(knownContentLength);
@@ -146,6 +152,7 @@ public void getArtifactBySHA1SanitizeEtag() {
final String knownMdBase16 = BaseEncoding.base16().lowerCase().encode(knownMd5.getBytes());
final String knownMd5Base64 = BaseEncoding.base64().encode(knownMd5.getBytes());
+ when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream);
when(amazonS3Mock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock);
when(s3ObjectMock.getObjectMetadata()).thenReturn(s3ObjectMetadataMock);
// add special characters to etag
diff --git a/licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt b/licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt
new file mode 100644
index 0000000..a1239d1
--- /dev/null
+++ b/licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt
@@ -0,0 +1,6 @@
+Copyright (c) 2021 Bosch.IO GmbH and others.
+
+All rights reserved. This program and the accompanying materials
+are made available under the terms of the Eclipse Public License v1.0
+which accompanies this distribution, and is available at
+http://www.eclipse.org/legal/epl-v10.html
diff --git a/pom.xml b/pom.xml
index 73e460e..23cc6a2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,12 +90,13 @@
com.mycila
license-maven-plugin
- licenses/LICENSE_HEADER_TEMPLATE_BOSCH_20.txt
+ licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt
licenses/LICENSE_HEADER_TEMPLATE_SIEMENS.txt
licenses/LICENSE_HEADER_TEMPLATE_SIEMENS_18.txt
licenses/LICENSE_HEADER_TEMPLATE_BOSCH.txt
licenses/LICENSE_HEADER_TEMPLATE_BOSCH_18.txt
+ licenses/LICENSE_HEADER_TEMPLATE_BOSCH_20.txt
licenses/LICENSE_HEADER_TEMPLATE_MICROSOFT_20.txt
licenses/LICENSE_HEADER_TEMPLATE_MICROSOFT_18.txt
licenses/LICENSE_HEADER_TEMPLATE_RICO_PAHLISCH.txt