Skip to content
This repository was archived by the owner on Jan 27, 2025. It is now read-only.

AWS SDK v2 support #64

Merged
Kirik0 merged 13 commits into from
Sep 14, 2022
Merged

AWS SDK v2 support #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
367a32b
Added OAuth support for membership fetching endpoint
Aug 1, 2022
46758ca
Updated docs for oauth config
Aug 1, 2022
4af592a
Fixed unit test for memberships fetch
Aug 1, 2022
f2c97e4
Merge branch 'master' of https://github.com/FINRAOS/Fidelius into mem…
Aug 1, 2022
c11e030
Additional security headers
Aug 5, 2022
7f1afb9
Made Content-Security-Policy header configurable
Aug 8, 2022
f25ae7a
Added Content-Security-Policy to UI
Aug 11, 2022
73d5bd5
Merge branch 'master' of https://github.com/FINRAOS/Fidelius into sec…
Aug 11, 2022
92ebff0
Updated Fidelius backend and SDK to use AWS SDK v2
Aug 31, 2022
7efeb33
Merge branch 'master' of https://github.com/FINRAOS/Fidelius into aws…
Sep 1, 2022
450d632
Small fix for source names fetched for non-RDS/Aurora source types
Sep 1, 2022
0d28667
Removed unused list
Sep 12, 2022
b4e2ac7
Updated Maven compiler plugin
Sep 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 36 additions & 20 deletions fidelius-sdk/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,44 +78,60 @@
</developers>
Copy link
Contributor

@SMxJrz SMxJrz Sep 9, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we update the maven compiler plugin to line up with this PR? #65


<properties>
<aws.version>1.11.767</aws.version>
<aws.version>2.16.60</aws.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${aws.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<dependency>
<artifactId>aws-java-sdk-core</artifactId>
<groupId>com.amazonaws</groupId>
<version>${aws.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>apache-client</artifactId>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>auth</artifactId>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>dynamodb</artifactId>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
<version>${aws.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>ec2</artifactId>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-kms</artifactId>
<version>${aws.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kms</artifactId>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-ec2</artifactId>
<version>${aws.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>lambda</artifactId>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sts</artifactId>
<version>${aws.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>rds</artifactId>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-lambda</artifactId>
<version>${aws.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>sts</artifactId>
</dependency>

<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
Expand Down Expand Up @@ -182,7 +198,7 @@
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
<version>3.10.1</version>
</dependency>

</dependencies>
Expand Down
24 changes: 12 additions & 12 deletions fidelius-sdk/src/main/java/org/finra/fidelius/CredModelMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.finra.fidelius;

import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;

import java.util.HashMap;
import java.util.Map;
Expand All @@ -31,28 +31,28 @@ private enum DynamoAttributes{

public static Map<String,AttributeValue> toDynamo(EncryptedCredential encryptedCredential){
HashMap<String, AttributeValue> dynamoRow = new HashMap<>();
dynamoRow.put(DynamoAttributes.name.name(), new AttributeValue(encryptedCredential.getFullName()));
dynamoRow.put(DynamoAttributes.version.name(), new AttributeValue(encryptedCredential.getVersion()));
dynamoRow.put(DynamoAttributes.key.name(), new AttributeValue(encryptedCredential.getDatakey()));
dynamoRow.put(DynamoAttributes.contents.name(), new AttributeValue(encryptedCredential.getCredential()));
dynamoRow.put(DynamoAttributes.hmac.name(), new AttributeValue(encryptedCredential.getHmac()));
dynamoRow.put(DynamoAttributes.name.name(), AttributeValue.builder().s(encryptedCredential.getFullName()).build());
dynamoRow.put(DynamoAttributes.version.name(), AttributeValue.builder().s(encryptedCredential.getVersion()).build());
dynamoRow.put(DynamoAttributes.key.name(), AttributeValue.builder().s(encryptedCredential.getDatakey()).build());
dynamoRow.put(DynamoAttributes.contents.name(), AttributeValue.builder().s(encryptedCredential.getCredential()).build());
dynamoRow.put(DynamoAttributes.hmac.name(), AttributeValue.builder().s(encryptedCredential.getHmac()).build());

if(encryptedCredential.getUpdateBy()!=null)
dynamoRow.put(DynamoAttributes.updatedBy.name(), new AttributeValue(encryptedCredential.getUpdateBy()));
dynamoRow.put(DynamoAttributes.updatedBy.name(), AttributeValue.builder().s(encryptedCredential.getUpdateBy()).build());

if(encryptedCredential.getUpdateOn()!=null)
dynamoRow.put(DynamoAttributes.updatedOn.name(), new AttributeValue(encryptedCredential.getUpdateOn()));
dynamoRow.put(DynamoAttributes.updatedOn.name(), AttributeValue.builder().s(encryptedCredential.getUpdateOn()).build());

if(encryptedCredential.getSdlc()!=null)
dynamoRow.put(DynamoAttributes.sdlc.name(), new AttributeValue(encryptedCredential.getSdlc()));
dynamoRow.put(DynamoAttributes.sdlc.name(), AttributeValue.builder().s(encryptedCredential.getSdlc()).build());

if(encryptedCredential.getComponent()!= null)
dynamoRow.put(DynamoAttributes.component.name(), new AttributeValue(encryptedCredential.getComponent()));
dynamoRow.put(DynamoAttributes.component.name(), AttributeValue.builder().s(encryptedCredential.getComponent()).build());

return dynamoRow;
}

public static EncryptedCredential fromDynamo(Map<String,AttributeValue> dynamoCred){
public static EncryptedCredential fromDynamo(Map<String, AttributeValue> dynamoCred){
return new EncryptedCredential()
.setFullName(getAttributeValue(DynamoAttributes.name.name(), dynamoCred))
.setCredential(getAttributeValue(DynamoAttributes.contents.name(),dynamoCred))
Expand All @@ -68,7 +68,7 @@ public static EncryptedCredential fromDynamo(Map<String,AttributeValue> dynamoCr
private static String getAttributeValue(String name, Map<String,AttributeValue> dynamoCred){
AttributeValue attributeValue = dynamoCred.get(name);
if(attributeValue!=null){
return attributeValue.getS();
return attributeValue.s();
}
return null;
}
Expand Down
155 changes: 86 additions & 69 deletions fidelius-sdk/src/main/java/org/finra/fidelius/FideliusClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,114 +17,128 @@

package org.finra.fidelius;

import java.util.Collections;
import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.regions.*;
import com.amazonaws.retry.PredefinedRetryPolicies;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
import com.amazonaws.services.ec2.model.*;
import com.amazonaws.services.lambda.AWSLambda;
import com.amazonaws.services.lambda.AWSLambdaClientBuilder;
import com.amazonaws.services.lambda.model.*;
import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.AWSKMSClientBuilder;
import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest;
import com.amazonaws.util.EC2MetadataUtils;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.awscore.retry.AwsRetryPolicy;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
import software.amazon.awssdk.core.internal.http.loader.DefaultSdkHttpClientBuilder;
import software.amazon.awssdk.http.SdkHttpClient;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
import software.amazon.awssdk.http.apache.internal.impl.ApacheSdkHttpClient;
import software.amazon.awssdk.regions.internal.util.EC2MetadataUtils;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder;
import software.amazon.awssdk.services.ec2.Ec2Client;
import software.amazon.awssdk.services.ec2.Ec2ClientBuilder;
import software.amazon.awssdk.services.ec2.model.*;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.KmsClientBuilder;
import software.amazon.awssdk.services.lambda.LambdaClient;
import software.amazon.awssdk.services.lambda.LambdaClientBuilder;
import software.amazon.awssdk.services.sts.StsClient;
import software.amazon.awssdk.services.sts.StsClientBuilder;
import software.amazon.awssdk.services.sts.model.GetCallerIdentityRequest;

import javax.swing.plaf.synth.Region;


public class FideliusClient {
private static final Logger logger = LoggerFactory.getLogger(FideliusClient.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();

protected EnvConfig envConfig;
protected ProxyConfiguration proxyConfig;
protected JCredStash jCredStash;
protected AWSSecurityTokenService awsSecurityTokenService;
protected StsClient stsClient;

private final AmazonEC2 client;
private final AWSLambda lambda;
private final Ec2Client ec2Client;
private final LambdaClient lambdaClient;

public FideliusClient() {
this(null, new DefaultAWSCredentialsProviderChain());
this(null, AwsCredentialsProviderChain.builder().addCredentialsProvider(DefaultCredentialsProvider.create()).build());
}

public FideliusClient(String region) {
this(null, new DefaultAWSCredentialsProviderChain(), region);
this(null, AwsCredentialsProviderChain.builder().addCredentialsProvider(DefaultCredentialsProvider.create()).build(), region);
}

public FideliusClient(ClientConfiguration clientConf, AWSCredentialsProvider provider) {
public FideliusClient(ClientOverrideConfiguration clientConf, AwsCredentialsProvider provider) {
this(clientConf, provider, null);
}

public FideliusClient(ClientConfiguration clientConf, AWSCredentialsProvider provider, String region) {
public FideliusClient(ClientOverrideConfiguration clientConf, AwsCredentialsProvider provider, String region) {

envConfig = new EnvConfig();
ClientConfiguration kmsEc2ClientConfiguration = clientConf;
ClientOverrideConfiguration kmsEc2ClientConfiguration = clientConf;

if(clientConf==null){
clientConf = defaultClientConfiguration(envConfig);
clientConf.setRetryPolicy(PredefinedRetryPolicies.DYNAMODB_DEFAULT);
kmsEc2ClientConfiguration = defaultClientConfiguration(envConfig);
kmsEc2ClientConfiguration.setRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(5));
}

AmazonDynamoDBClientBuilder ddbBuilder = AmazonDynamoDBClientBuilder.standard()
.withCredentials(provider)
.withClientConfiguration(clientConf);
DynamoDbClientBuilder dynamoDbBuilder = DynamoDbClient.builder()
.credentialsProvider(provider)
.overrideConfiguration(clientConf);

AWSKMSClientBuilder kmsBuilder = AWSKMSClientBuilder.standard()
.withCredentials(provider)
.withClientConfiguration(kmsEc2ClientConfiguration);
KmsClientBuilder kmsBuilder = KmsClient.builder()
.credentialsProvider(provider)
.overrideConfiguration(kmsEc2ClientConfiguration);

AWSSecurityTokenServiceClientBuilder stsBuilder = AWSSecurityTokenServiceClientBuilder.standard()
.withClientConfiguration(clientConf)
.withCredentials(provider);
StsClientBuilder stsBuilder = StsClient.builder()
.overrideConfiguration(clientConf)
.credentialsProvider(provider);

AmazonEC2ClientBuilder clientBuilder = AmazonEC2ClientBuilder.standard()
.withCredentials(provider)
.withClientConfiguration(kmsEc2ClientConfiguration);
Ec2ClientBuilder clientBuilder = Ec2Client.builder()
.credentialsProvider(provider)
.overrideConfiguration(kmsEc2ClientConfiguration);

AWSLambdaClientBuilder lambdaClientBuilder = AWSLambdaClientBuilder.standard()
.withClientConfiguration(clientConf)
.withCredentials(provider);
LambdaClientBuilder lambdaClientBuilder = LambdaClient.builder()
.credentialsProvider(provider)
.overrideConfiguration(clientConf);

if(region != null){
Regions regionEnum = Regions.fromName(region);
ddbBuilder.withRegion(regionEnum);
kmsBuilder.withRegion(regionEnum);
stsBuilder.withRegion(regionEnum);
clientBuilder.withRegion(regionEnum);
lambdaClientBuilder.withRegion(regionEnum);
software.amazon.awssdk.regions.Region awsRegion = software.amazon.awssdk.regions.Region.of(region);
dynamoDbBuilder = dynamoDbBuilder.region(awsRegion);
kmsBuilder = kmsBuilder.region(awsRegion);
stsBuilder = stsBuilder.region(awsRegion);
clientBuilder = clientBuilder.region(awsRegion);
lambdaClientBuilder = lambdaClientBuilder.region(awsRegion);
}
lambda = lambdaClientBuilder.build();
client = clientBuilder.build();
awsSecurityTokenService = stsBuilder.build();
jCredStash = new JCredStash(ddbBuilder.build(), kmsBuilder.build(), awsSecurityTokenService);
if(envConfig.hasProxyEnv()) {
Copy link
Member

@Brijeshrpatel9 Brijeshrpatel9 Sep 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if I understand this if condition, don't we have to initialize all the clients if proxy env is provided and if not provided, like if.....else....

Copy link
Contributor Author

@Kirik0 Kirik0 Sep 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're still building the Clients on lines 128-131. For AWS SDK v2, client builders like the DynamoDbClientBuilder need an additional HTTP Client built-in for proxy configuration to work. The if-block there takes care of that if proxy configuration is provided but skips if it's not.

SdkHttpClient sdkHttpClient = ApacheHttpClient.builder()
.proxyConfiguration(proxyConfig).build();
dynamoDbBuilder = dynamoDbBuilder.httpClient(sdkHttpClient);
kmsBuilder = kmsBuilder.httpClient(sdkHttpClient);
stsBuilder = stsBuilder.httpClient(sdkHttpClient);
clientBuilder = clientBuilder.httpClient(sdkHttpClient);
lambdaClientBuilder = lambdaClientBuilder.httpClient(sdkHttpClient);
}
lambdaClient = lambdaClientBuilder.build();
ec2Client = clientBuilder.build();
stsClient = stsBuilder.build();
jCredStash = new JCredStash(dynamoDbBuilder.build(), kmsBuilder.build(), stsClient);
}

protected void setFideliusClient(AmazonDynamoDB ddb, AWSKMS kms) {
jCredStash = new JCredStash(ddb, kms, awsSecurityTokenService);
protected void setFideliusClient(DynamoDbClient ddb, KmsClient kms) {
jCredStash = new JCredStash(ddb, kms, stsClient);
}

protected ClientConfiguration defaultClientConfiguration(EnvConfig envConfig){
ClientConfiguration clientConfiguration = new ClientConfiguration();
protected ClientOverrideConfiguration defaultClientConfiguration(EnvConfig envConfig){
ClientOverrideConfiguration clientConfiguration = ClientOverrideConfiguration.builder().retryPolicy(AwsRetryPolicy.defaultRetryPolicy()).build();
if(envConfig.hasProxyEnv()) {
clientConfiguration.setProxyHost(envConfig.getProxy());
clientConfiguration.setProxyPort(Integer.parseInt(envConfig.getPort()));
proxyConfig = ProxyConfiguration.builder().endpoint(URI.create(envConfig.getProxy() + ":" + envConfig.getPort())).build();
}
return clientConfiguration;
}
Expand Down Expand Up @@ -161,23 +175,23 @@ protected HashMap<String, String> getEC2Tags() {

String instanceID = EC2MetadataUtils.getInstanceId();

DescribeInstancesRequest instancesRequest = new DescribeInstancesRequest().withInstanceIds(instanceID);
DescribeInstancesResult instancesResult = client.describeInstances(instancesRequest);
DescribeInstancesRequest instancesRequest = DescribeInstancesRequest.builder().instanceIds(instanceID).build();
DescribeInstancesResponse instancesResult = ec2Client.describeInstances(instancesRequest);

// There should only be one Instance with identical instanceID
List<Reservation> reservations = instancesResult.getReservations();
List<Reservation> reservations = instancesResult.reservations();
if (reservations.size() > 1) {
return null;
}

Reservation reservation = reservations.get(0);
Instance instance = reservation.getInstances().get(0);
List<Tag> tagList = instance.getTags();
Instance instance = reservation.instances().get(0);
List<Tag> tagList = instance.tags();

HashMap<String, String> tagMap = new HashMap<String, String>();
for (Tag t : tagList) {
if (t.getKey().equals(Constants.FID_CONTEXT_APPLICATION) || t.getKey().equals(Constants.FID_CONTEXT_SDLC) || t.getKey().equals(Constants.FID_CONTEXT_COMPONENT))
tagMap.put(t.getKey(), t.getValue());
if (t.key().equals(Constants.FID_CONTEXT_APPLICATION) || t.key().equals(Constants.FID_CONTEXT_SDLC) || t.key().equals(Constants.FID_CONTEXT_COMPONENT))
tagMap.put(t.key(), t.value());
}
return tagMap;
}
Expand Down Expand Up @@ -245,7 +259,7 @@ protected String getUser() throws Exception {
}

protected String getUserIdentity() throws Exception {
return awsSecurityTokenService.getCallerIdentity(new GetCallerIdentityRequest()).getArn();
return stsClient.getCallerIdentity(GetCallerIdentityRequest.builder().build()).arn();
}

/**
Expand Down Expand Up @@ -507,6 +521,9 @@ protected void deleteCredential(String name, String application, String sdlc,
logger.info("User "+ user + " deleted credential " + prefixedName);
} catch (RuntimeException e) { // Credential not found
logger.info("Credential " + prefixedName + " not found. [" + e.toString() + "] ");
for(StackTraceElement ste : e.getStackTrace()) {
logger.error(ste.toString());
}
throw new RuntimeException(e);
}
}
Expand Down
Loading