From e9a8f6d97a0f76dfcfc9bf0e67edcfb443d462ba Mon Sep 17 00:00:00 2001 From: "angel.herce" Date: Thu, 24 Apr 2025 12:32:22 +0200 Subject: [PATCH] performance: S3 Connection has been optimized --- CHANGELOG.md | 3 ++ .../s3/repository/OSdmsS3Repository.java | 17 +++++++---- .../config/OSdmsS3RepositoryConfig.java | 28 ++++++++++++++++--- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3807ab..86b9b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ ## [Unreleased] +### Changed 🛠️ +* **S3 Connection:** The connection to S3 has been optimized. + ## [1.3.1] - 2023-06-28 ### Fixed 🐛 diff --git a/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/OSdmsS3Repository.java b/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/OSdmsS3Repository.java index a546f50..95b0611 100644 --- a/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/OSdmsS3Repository.java +++ b/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/OSdmsS3Repository.java @@ -15,6 +15,7 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -143,12 +144,15 @@ public OSdmsS3RepositoryResponse download( final ListObjec final List findResponseData = findResponse.getData(); findResponseData.forEach( target -> { final GetObjectRequest getObjectRequest = new GetObjectRequest( target.getBucket(), target.getKey() ); - final S3Object s3Object = this.amazonS3.getObject( getObjectRequest ); - - if( s3Object != null ) { - final OSdmsS3RepositoryDto dto = new OSdmsS3RepositoryDto(); - dto.set( s3Object ); - data.add( dto ); + try( final S3Object s3Object = this.amazonS3.getObject( getObjectRequest )){ + if( s3Object != null ) { + final OSdmsS3RepositoryDto dto = new OSdmsS3RepositoryDto(); + dto.set( s3Object ); + data.add( dto ); + } + } + catch( final IOException e ){ + LOGGER.error("Error closing S3Object stream: {}", e.getMessage()); } } ); @@ -200,6 +204,7 @@ public OSdmsS3RepositoryResponse upload( final PutObjectRe try { final Upload upload = transferManager.upload( request ); upload.waitForCompletion(); + transferManager.shutdownNow(false); final ListObjectsRequest findRequest = new ListObjectsRequest() .withBucketName( request.getBucketName() ) diff --git a/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/config/OSdmsS3RepositoryConfig.java b/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/config/OSdmsS3RepositoryConfig.java index 65d7179..bdde0c0 100644 --- a/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/config/OSdmsS3RepositoryConfig.java +++ b/ontimize-jee-sdms-engine/ontimize-jee-sdms-engine-s3/src/main/java/com/ontimize/jee/sdms/engine/s3/repository/config/OSdmsS3RepositoryConfig.java @@ -1,5 +1,6 @@ package com.ontimize.jee.sdms.engine.s3.repository.config; +import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; @@ -24,21 +25,30 @@ public class OSdmsS3RepositoryConfig { @Value( "${ontimize.sdms.s3.access-key}" ) private String accessKey; - /** The secret key from Amazon S3 */ @Value( "${ontimize.sdms.s3.secret-key}" ) private String secretKey; - /** The region of Amazon S3 */ @Value( "${ontimize.sdms.s3.region}" ) private String region; - /** The endpoing of Amazon S3 */ @Value( "${ontimize.sdms.s3.endpoint:}" ) private String endpoint; + @Value( "${ontimize.sdms.s3.max-connections:100}" ) + private int maxConnections; + + @Value( "${ontimize.sdms.s3.timeout.connection:5000}" ) + private int connectionTimeout; + + @Value( "${ontimize.sdms.s3.timeout.socket:10000}" ) + private int socketTimeout; + + @Value( "${ontimize.sdms.s3.ttl-connection:60000}" ) + private long ttlConnection; + // ------------------------------------------------------------------------------------------------------------------ \\ /** @@ -51,9 +61,19 @@ public AmazonS3 amazonS3() { //Initialise AWS credentials BasicAWSCredentials awsCreds = new BasicAWSCredentials( this.accessKey, this.secretKey ); + ClientConfiguration clientConfig = new ClientConfiguration() + .withMaxConnections(this.maxConnections) // Aumenta el número de conexiones simultáneas + .withConnectionTimeout(this.connectionTimeout) // Tiempo máximo para establecer la conexión (ms) + .withSocketTimeout(this.socketTimeout) // Tiempo máximo para leer datos del socket (ms) + .withTcpKeepAlive(true) // Mantiene las conexiones abiertas + .withUseExpectContinue(true) // Mejora rendimiento en PUTs grandes + .withConnectionTTL(this.ttlConnection) // TTL de conexión, útil para liberar sockets antiguos + .withMaxErrorRetry(3); + //Configure and return AmazonS3 bean final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard() - .withCredentials( new AWSStaticCredentialsProvider( awsCreds ) ); + .withClientConfiguration( clientConfig ) + .withCredentials( new AWSStaticCredentialsProvider( awsCreds )); if( this.endpoint != null && ! this.endpoint.isEmpty() ) { builder.withEndpointConfiguration(