Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Replicated from snowflake-jdbc (v3.25.1)
* Source: https://github.com/snowflakedb/snowflake-jdbc/blob/v3.25.1/src/main/java/net/snowflake/client/jdbc/cloud/storage/AwsSdkGCPSigner.java
*/
package net.snowflake.ingest.streaming.internal.fileTransferAgent;

import com.amazonaws.SignableRequest;
import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.http.HttpMethodName;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;

public class AwsSdkGCPSigner extends AWS4Signer {
private static final Map<String, String> headerMap =
new HashMap<String, String>() {
{
put("x-amz-storage-class", "x-goog-storage-class");
put("x-amz-acl", "x-goog-acl");
put("x-amz-date", "x-goog-date");
put("x-amz-copy-source", "x-goog-copy-source");
put("x-amz-metadata-directive", "x-goog-metadata-directive");
put("x-amz-copy-source-if-match", "x-goog-copy-source-if-match");
put("x-amz-copy-source-if-none-match", "x-goog-copy-source-if-none-match");
put("x-amz-copy-source-if-unmodified-since", "x-goog-copy-source-if-unmodified-since");
put("x-amz-copy-source-if-modified-since", "x-goog-copy-source-if-modified-since");
}
};

@Override
public void sign(SignableRequest<?> request, AWSCredentials credentials) {
if (credentials.getAWSAccessKeyId() != null && !"".equals(credentials.getAWSAccessKeyId())) {
request.addHeader("Authorization", "Bearer " + credentials.getAWSAccessKeyId());
}

if (request.getHttpMethod() == HttpMethodName.GET) {
request.addHeader("Accept-Encoding", "gzip,deflate");
}

Map<String, String> headerCopy =
request.getHeaders().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

for (Map.Entry<String, String> entry : headerCopy.entrySet()) {
String entryKey = entry.getKey().toLowerCase();
if (headerMap.containsKey(entryKey)) {
request.addHeader(headerMap.get(entryKey), entry.getValue());
} else if (entryKey.startsWith("x-amz-meta-")) {
request.addHeader(entryKey.replace("x-amz-meta-", "x-goog-meta-"), entry.getValue());
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,10 @@ class GCSAccessStrategyAwsSdk implements GCSAccessStrategy {
ClientConfiguration clientConfig = new ClientConfiguration();

SignerFactory.registerSigner(
"net.snowflake.client.jdbc.cloud.storage.AwsSdkGCPSigner",
net.snowflake.client.jdbc.cloud.storage.AwsSdkGCPSigner.class);
clientConfig.setSignerOverride("net.snowflake.client.jdbc.cloud.storage.AwsSdkGCPSigner");
"net.snowflake.ingest.streaming.internal.fileTransferAgent.AwsSdkGCPSigner",
AwsSdkGCPSigner.class);
clientConfig.setSignerOverride(
"net.snowflake.ingest.streaming.internal.fileTransferAgent.AwsSdkGCPSigner");

clientConfig
.getApacheHttpClientConfig()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Replicated from snowflake-jdbc: net.snowflake.client.core.HttpClientSettingsKey
* Replicated from snowflake-jdbc: HttpClientSettingsKey
* Tag: v3.25.1
* Source: https://github.com/snowflakedb/snowflake-jdbc/blob/v3.25.1/src/main/java/net/snowflake/client/core/HttpClientSettingsKey.java
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,10 @@ public class S3HttpUtil {
*/
// Parameter uses JDBC's HttpClientSettingsKey because session.getHttpClientKey() returns it.
// This path is only used when session != null (never from streaming ingest).
public static void setProxyForS3(
net.snowflake.client.core.HttpClientSettingsKey key, ClientConfiguration clientConfig) {
public static void setProxyForS3(HttpClientSettingsKey key, ClientConfiguration clientConfig) {
if (key != null && key.usesProxy()) {
clientConfig.setProxyProtocol(
key.getProxyHttpProtocol() == net.snowflake.client.core.HttpProtocol.HTTPS
? Protocol.HTTPS
: Protocol.HTTP);
key.getProxyHttpProtocol() == HttpProtocol.HTTPS ? Protocol.HTTPS : Protocol.HTTP);
clientConfig.setProxyHost(key.getProxyHost());
clientConfig.setProxyPort(key.getProxyPort());
clientConfig.setNonProxyHosts(key.getNonProxyHosts());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -638,9 +638,9 @@ public static void uploadWithoutConnection(SnowflakeFileTransferConfig config) t
String streamingIngestClientKey = config.getStreamingIngestClientKey();

// Create HttpClient key
net.snowflake.client.core.HttpClientSettingsKey key =
net.snowflake.client.jdbc.SnowflakeUtil.convertProxyPropertiesToHttpClientKey(
net.snowflake.client.core.OCSPMode.FAIL_OPEN, proxyProperties);
HttpClientSettingsKey key =
StorageClientUtil.convertProxyPropertiesToHttpClientKey(
net.snowflake.ingest.utils.OCSPMode.FAIL_OPEN, proxyProperties);

StageInfo stageInfo = metadata.getStageInfo();
stageInfo.setProxyProperties(proxyProperties);
Expand Down Expand Up @@ -860,7 +860,7 @@ private static void pushFileToRemoteStoreWithPresignedUrl(
FileCompressionType compressionType,
SnowflakeStorageClient initialClient,
int networkTimeoutInMilli,
net.snowflake.client.core.HttpClientSettingsKey ocspModeAndProxyKey,
HttpClientSettingsKey ocspModeAndProxyKey,
int parallel,
File srcFile,
boolean uploadFromStream,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ public InputStream downloadToStream(
@Override
public void uploadWithPresignedUrlWithoutConnection(
int networkTimeoutInMilli,
net.snowflake.client.core.HttpClientSettingsKey ocspModeAndProxyKey,
HttpClientSettingsKey ocspModeAndProxyKey,
int parallelism,
boolean uploadFromStream,
String remoteStorageLocation,
Expand Down Expand Up @@ -830,7 +830,7 @@ private void uploadWithPresignedUrl(
Map<String, String> metadata,
InputStream content,
String presignedUrl,
net.snowflake.client.core.HttpClientSettingsKey ocspAndProxyKey,
HttpClientSettingsKey ocspAndProxyKey,
String queryId)
throws SnowflakeSQLException {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ void upload(
@Deprecated
default void uploadWithPresignedUrlWithoutConnection(
int networkTimeoutInMilli,
net.snowflake.client.core.HttpClientSettingsKey ocspModeAndProxyKey,
HttpClientSettingsKey ocspModeAndProxyKey,
int parallelism,
boolean uploadFromStream,
String remoteStorageLocation,
Expand Down Expand Up @@ -291,7 +291,7 @@ default void uploadWithPresignedUrlWithoutConnection(
*/
default void uploadWithPresignedUrlWithoutConnection(
int networkTimeoutInMilli,
net.snowflake.client.core.HttpClientSettingsKey ocspModeAndProxyKey,
HttpClientSettingsKey ocspModeAndProxyKey,
int parallelism,
boolean uploadFromStream,
String remoteStorageLocation,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,22 +234,21 @@ static void throwJCEMissingError(String operation, Exception ex, String queryId)
* Replicated from SnowflakeFileTransferAgent.throwNoSpaceLeftError. Source:
* https://github.com/snowflakedb/snowflake-jdbc/blob/v3.25.1/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java
*
* @deprecated use {@link #throwNoSpaceLeftError(net.snowflake.client.core.SFSession, String,
* Exception, String)}
* @deprecated use {@link #throwNoSpaceLeftError(Object, String, Exception, String)}
*/
@Deprecated
static void throwNoSpaceLeftError(
net.snowflake.client.core.SFSession session, String operation, Exception ex)
static void throwNoSpaceLeftError(Object session, String operation, Exception ex)
throws SnowflakeSQLLoggedException {
throwNoSpaceLeftError(session, operation, ex, null);
}

/**
* Replicated from SnowflakeFileTransferAgent.throwNoSpaceLeftError. Source:
* https://github.com/snowflakedb/snowflake-jdbc/blob/v3.25.1/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java
*
* <p>Note: session parameter is always null from ingest callers. Kept for API shape.
*/
static void throwNoSpaceLeftError(
net.snowflake.client.core.SFSession session, String operation, Exception ex, String queryId)
static void throwNoSpaceLeftError(Object session, String operation, Exception ex, String queryId)
throws SnowflakeSQLLoggedException {
String exMessage = getRootCause(ex).getMessage();
if (exMessage != null && exMessage.equals(NO_SPACE_LEFT_ON_DEVICE_ERR)) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/net/snowflake/ingest/utils/OCSPMode.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Replicated from snowflake-jdbc: net.snowflake.client.core.OCSPMode
* Replicated from snowflake-jdbc: net.snowflake.ingest.utils.OCSPMode
* Tag: v3.25.1
* Source: https://github.com/snowflakedb/snowflake-jdbc/blob/v3.25.1/src/main/java/net/snowflake/client/core/OCSPMode.java
*
Expand Down
Loading