Skip to content

Commit c2c1308

Browse files
ggcrggcr
committed
GetObject basic implementation
1 parent 029aff1 commit c2c1308

5 files changed

Lines changed: 141 additions & 71 deletions

File tree

‎src/s3cpp/auth.h‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class AWSSigV4Signer {
6060
}
6161

6262
std::string createCannonicalRequest(HttpRequest& request) {
63-
const std::string http_verb = getHttpVerb(request.getHttpMethod());
63+
const std::string http_verb = request.getHttpMethodStr(request.getHttpMethod());
6464
std::string url = request.getURL();
6565

6666
// URI
@@ -177,24 +177,6 @@ class AWSSigV4Signer {
177177
std::string secret_key;
178178
std::string aws_region;
179179

180-
// Cannonicalize HTTP verb from the request
181-
std::string getHttpVerb(const HttpMethod& http_method) {
182-
switch (http_method) {
183-
case HttpMethod::Get:
184-
return "GET";
185-
case HttpMethod::Head:
186-
return "HEAD";
187-
case HttpMethod::Post:
188-
return "POST";
189-
case HttpMethod::Put:
190-
return "PUT";
191-
case HttpMethod::Delete:
192-
return "DELETE";
193-
default:
194-
throw std::runtime_error("No known Http Method");
195-
}
196-
}
197-
198180
const unsigned char* deriveSigningKey(const std::string request_date) {
199181
const std::string initial_candidate = "AWS4" + secret_key;
200182
const unsigned char* keyCandidate = reinterpret_cast<const unsigned char*>(initial_candidate.c_str());

‎src/s3cpp/httpclient.h‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,26 @@ class HttpRequestBase {
9797
return headers_;
9898
}
9999

100+
// Cannonicalize HTTP verb from the request
101+
const std::string getHttpMethodStr(const HttpMethod& http_method) const {
102+
switch (http_method) {
103+
case HttpMethod::Get:
104+
return "GET";
105+
case HttpMethod::Head:
106+
return "HEAD";
107+
case HttpMethod::Post:
108+
return "POST";
109+
case HttpMethod::Put:
110+
return "PUT";
111+
case HttpMethod::Delete:
112+
return "DELETE";
113+
default:
114+
throw std::runtime_error("No known Http Method");
115+
}
116+
}
117+
118+
119+
100120
protected:
101121
HttpClient& client_;
102122
std::string URL_;

‎src/s3cpp/s3.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
ListBucketResult S3Client::ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken) {
44
// Silent-ly accept maxKeys > 1000, even though we will return 1K at most
55
// Pagination is opt-in as in the Go SDK, the user must be aware of this
6+
const std::string baseUrl = buildURL(bucket);
67
std::string url;
78
if (continuationToken.size() > 0) {
8-
url = std::format("http://127.0.0.1:9000/{}?list-type=2&prefix={}&max-keys={}&continuation-token={}", bucket, prefix, maxKeys, continuationToken);
9+
url = baseUrl + std::format("?list-type=2&prefix={}&max-keys={}&continuation-token={}", prefix, maxKeys, continuationToken);
910
} else {
10-
url = std::format("http://127.0.0.1:9000/{}?list-type=2&prefix={}&max-keys={}", bucket, prefix, maxKeys);
11+
url = baseUrl + std::format("?list-type=2&prefix={}&max-keys={}", prefix, maxKeys);
1112
}
12-
HttpRequest req = Client.get(url).header("Host", "127.0.0.1");
13+
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
1314
Signer.sign(req);
1415
HttpResponse res = req.execute();
1516
ListBucketResult response = deserializeListBucketResult(Parser.parse(res.body()), maxKeys);

‎src/s3cpp/s3.h‎

Lines changed: 60 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,33 +47,81 @@ struct ListBucketResult {
4747
std::string StartAfter;
4848
};
4949

50+
enum class S3AddressingStyle {
51+
VirtualHosted,
52+
PathStyle
53+
};
54+
5055
class S3Client {
5156
public:
5257
// TODO(cristian): We should accept and define the endpoint url here
5358
S3Client(const std::string& access, const std::string& secret)
5459
: Client(HttpClient())
5560
, Signer(AWSSigV4Signer(access, secret))
56-
, Parser(XMLParser()) { }
61+
, Parser(XMLParser())
62+
, addressing_style_(S3AddressingStyle::VirtualHosted) {
63+
// When no endpoint is provided we default to us-east-1 (flashbacks from vietnam)
64+
endpoint_ = std::format("s3.us-east-1.amazonaws.com");
65+
}
5766
S3Client(const std::string& access, const std::string& secret, const std::string& region)
5867
: Client(HttpClient())
5968
, Signer(AWSSigV4Signer(access, secret, region))
60-
, Parser(XMLParser()) { }
69+
, Parser(XMLParser())
70+
, addressing_style_(S3AddressingStyle::VirtualHosted) {
71+
// When no endpoint is provided we default to AWS
72+
endpoint_ = std::format("s3.{}.amazonaws.com", region); // TODO(cristian): Ping to validate region
73+
}
74+
S3Client(const std::string& access, const std::string& secret, const std::string& customEndpoint, S3AddressingStyle style)
75+
: Client(HttpClient())
76+
, Signer(AWSSigV4Signer(access, secret))
77+
, Parser(XMLParser())
78+
, endpoint_(customEndpoint)
79+
, addressing_style_(style) {
80+
}
6181

62-
ListBucketResult GetObject(const std::string& bucket, const std::string& key) {
63-
return ListBucketResult{};
64-
}
82+
// TODO(cristian): Implement deserialization
83+
// TODO(cristian): Wrap onto std::expected
84+
ListBucketResult GetObject(const std::string& bucket, const std::string& key) {
85+
std::string url = buildURL(bucket) + std::format("/{}", key);
86+
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
87+
Signer.sign(req);
88+
HttpResponse res = req.execute();
89+
std::println("{}", res.body());
90+
ListBucketResult response = deserializeListBucketResult(Parser.parse(res.body()), 1000);
91+
return response;
92+
}
6593

66-
ListBucketResult ListObjects(const std::string& bucket) { return ListObjects(bucket, "/", 1000, ""); }
67-
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix) { return ListObjects(bucket, prefix, 1000, ""); }
68-
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys) { return ListObjects(bucket, prefix, maxKeys, ""); }
69-
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken);
94+
ListBucketResult ListObjects(const std::string& bucket) { return ListObjects(bucket, "/", 1000, ""); }
95+
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix) { return ListObjects(bucket, prefix, 1000, ""); }
96+
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys) { return ListObjects(bucket, prefix, maxKeys, ""); }
97+
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken);
7098

71-
ListBucketResult deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys);
99+
ListBucketResult deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys);
72100

73101
private:
74-
HttpClient Client;
75-
AWSSigV4Signer Signer;
102+
HttpClient Client;
103+
AWSSigV4Signer Signer;
76104
XMLParser Parser;
105+
std::string endpoint_;
106+
S3AddressingStyle addressing_style_;
107+
108+
std::string buildURL(const std::string& bucket) const {
109+
if (addressing_style_ == S3AddressingStyle::VirtualHosted) {
110+
// bucket.s3.region.amazonaws.com/key
111+
return std::format("https://{}.{}", bucket, endpoint_);
112+
} else {
113+
// endpoint/bucket/key
114+
return std::format("http://{}/{}", endpoint_, bucket);
115+
}
116+
}
117+
118+
std::string getHostHeader(const std::string& bucket) const {
119+
if (addressing_style_ == S3AddressingStyle::VirtualHosted) {
120+
return std::format("{}.{}", bucket, endpoint_);
121+
} else {
122+
return endpoint_;
123+
}
124+
}
77125
};
78126

79127
class ListObjectsPaginator {

‎test/s3_test.cpp‎

Lines changed: 56 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
#include <s3cpp/s3.h>
33

44
TEST(S3, ListObjectsNoPrefix) {
5-
S3Client client("minio_access", "minio_secret");
5+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
66
try {
7-
client.ListObjects("my-bucket");
7+
// Assuming the bucket has the 10K objects
8+
// Once we implement PutObject we will do this ourselves with s3cpp
9+
ListBucketResult res = client.ListObjects("my-bucket");
10+
EXPECT_EQ(res.Contents.size(), 0);
811
} catch (const std::exception& e) {
912
const std::string emsg = e.what();
1013
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -15,9 +18,11 @@ TEST(S3, ListObjectsNoPrefix) {
1518
}
1619

1720
TEST(S3, ListObjectsFilePrefix) {
18-
S3Client client("minio_access", "minio_secret");
21+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
1922
try {
20-
client.ListObjects("my-bucket", "path/to/file.txt");
23+
// path/to/file_1.txt must exist
24+
ListBucketResult res = client.ListObjects("my-bucket", "path/to/file_1.txt");
25+
EXPECT_EQ(res.Contents.size(), 1);
2126
} catch (const std::exception& e) {
2227
const std::string emsg = e.what();
2328
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -28,9 +33,11 @@ TEST(S3, ListObjectsFilePrefix) {
2833
}
2934

3035
TEST(S3, ListObjectsDirPrefix) {
31-
S3Client client("minio_access", "minio_secret");
36+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
3237
try {
33-
client.ListObjects("my-bucket", "path/to/", 100);
38+
// Get 100 keys
39+
ListBucketResult res = client.ListObjects("my-bucket", "path/to/", 100);
40+
EXPECT_EQ(res.Contents.size(), 100);
3441
} catch (const std::exception& e) {
3542
const std::string emsg = e.what();
3643
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -41,9 +48,10 @@ TEST(S3, ListObjectsDirPrefix) {
4148
}
4249

4350
TEST(S3, ListObjectsDirPrefixMaxKeys) {
44-
S3Client client("minio_access", "minio_secret");
51+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
4552
try {
46-
client.ListObjects("my-bucket", "path/to/", 1);
53+
ListBucketResult res = client.ListObjects("my-bucket", "path/to/", 1);
54+
EXPECT_EQ(res.Contents.size(), 1);
4755
} catch (const std::exception& e) {
4856
const std::string emsg = e.what();
4957
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -54,30 +62,27 @@ TEST(S3, ListObjectsDirPrefixMaxKeys) {
5462
}
5563

5664
TEST(S3, ListObjectsCheckFields) {
57-
S3Client client("minio_access", "minio_secret");
65+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
5866
try {
59-
ListBucketResult response = client.ListObjects("my-bucket", "path/to/", 2);
60-
61-
// Check top-level fields
62-
EXPECT_EQ(response.Name, "my-bucket");
63-
EXPECT_EQ(response.Prefix, "path/to/");
64-
EXPECT_EQ(response.MaxKeys, 2);
65-
EXPECT_EQ(response.IsTruncated, true);
66-
EXPECT_FALSE(response.NextContinuationToken.empty()); // V2 uses NextContinuationToken
67-
68-
// Should have exactly 2 contents
69-
EXPECT_EQ(response.Contents.size(), 2);
70-
71-
// Check first object
72-
EXPECT_EQ(response.Contents[0].Key, "path/to/file_1.txt");
73-
EXPECT_EQ(response.Contents[0].Size, 26);
74-
// Note: V2 doesn't return Owner by default (need fetch-owner=true)
75-
EXPECT_EQ(response.Contents[0].StorageClass, "STANDARD");
76-
77-
// Check second object
78-
EXPECT_EQ(response.Contents[1].Key, "path/to/file_10.txt");
79-
EXPECT_EQ(response.Contents[1].Size, 27);
80-
EXPECT_EQ(response.Contents[1].StorageClass, "STANDARD");
67+
ListBucketResult res = client.ListObjects("my-bucket", "path/to/", 2);
68+
69+
EXPECT_EQ(res.Name, "my-bucket");
70+
EXPECT_EQ(res.Prefix, "path/to/");
71+
EXPECT_EQ(res.MaxKeys, 2);
72+
EXPECT_EQ(res.IsTruncated, true);
73+
EXPECT_FALSE(res.NextContinuationToken.empty());
74+
75+
// Should have exactly 2 keys
76+
EXPECT_EQ(res.Contents.size(), 2);
77+
78+
EXPECT_EQ(res.Contents[0].Key, "path/to/file_1.txt");
79+
EXPECT_EQ(res.Contents[0].Size, 26);
80+
EXPECT_EQ(res.Contents[0].StorageClass, "STANDARD");
81+
82+
EXPECT_EQ(res.Contents[1].Key, "path/to/file_10.txt");
83+
EXPECT_EQ(res.Contents[1].Size, 27);
84+
EXPECT_EQ(res.Contents[1].StorageClass, "STANDARD");
85+
8186
} catch (const std::exception& e) {
8287
const std::string emsg = e.what();
8388
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -88,11 +93,11 @@ TEST(S3, ListObjectsCheckFields) {
8893
}
8994

9095
TEST(S3, ListObjectsCheckLenKeys) {
91-
S3Client client("minio_access", "minio_secret");
96+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
9297
try {
93-
// has 10K objects - limit is 1000 keys
98+
// has 10K objects - limit is 1000 keys
9499
ListBucketResult response = client.ListObjects("my-bucket", "path/to/");
95-
EXPECT_EQ(response.Contents.size(), 1000);
100+
EXPECT_EQ(response.Contents.size(), 1000);
96101
} catch (const std::exception& e) {
97102
const std::string emsg = e.what();
98103
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -103,7 +108,7 @@ TEST(S3, ListObjectsCheckLenKeys) {
103108
}
104109

105110
TEST(S3, ListObjectsPaginator) {
106-
S3Client client("minio_access", "minio_secret");
111+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
107112
try {
108113
// has 10K objects - fetch 100 per page
109114
ListObjectsPaginator paginator(client, "my-bucket", "path/to/", 100);
@@ -114,7 +119,8 @@ TEST(S3, ListObjectsPaginator) {
114119
while (paginator.HasMorePages()) {
115120
ListBucketResult page = paginator.NextPage();
116121
totalObjects += page.Contents.size();
117-
pageCount++;
122+
if (page.Contents.size() > 0)
123+
pageCount++;
118124

119125
if (paginator.HasMorePages()) {
120126
EXPECT_EQ(page.Contents.size(), 100);
@@ -127,7 +133,20 @@ TEST(S3, ListObjectsPaginator) {
127133
} catch (const std::exception& e) {
128134
const std::string emsg = e.what();
129135
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
130-
GTEST_SKIP_("Skipping MinIOBasicRequest: Server not up");
136+
GTEST_SKIP_("Skipping ListObjectsPaginator: Server not up");
137+
}
138+
throw;
139+
}
140+
}
141+
142+
TEST(S3, GetObjectExists) {
143+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
144+
try {
145+
ListBucketResult response = client.GetObject("my-bucket", "path/to/file_1.txt");
146+
} catch (const std::exception& e) {
147+
const std::string emsg = e.what();
148+
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
149+
GTEST_SKIP_("Skipping GetObjectExists: Server not up");
131150
}
132151
throw;
133152
}

0 commit comments

Comments
 (0)