Skip to content

Commit b4bc60d

Browse files
ggcrggcr
committed
Support opts on ListObjectsV2 and GetObject
1 parent 3854c58 commit b4bc60d

4 files changed

Lines changed: 244 additions & 119 deletions

File tree

‎src/s3cpp/s3.cpp‎

Lines changed: 51 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,48 @@
11
#include <expected>
22
#include <s3cpp/s3.h>
33

4-
std::expected<ListObjectsResult, Error> S3Client::ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken) {
4+
std::expected<ListObjectsResult, Error> S3Client::ListObjects(const std::string& bucket, const ListObjectsInput& options) {
55
// Silent-ly accept maxKeys > 1000, even though we will return 1K at most
66
// Pagination is opt-in as in the Go SDK, the user must be aware of this
7-
const std::string baseUrl = buildURL(bucket);
8-
std::string url;
9-
if (continuationToken.size() > 0) {
10-
url = baseUrl + std::format("?list-type=2&prefix={}&max-keys={}&continuation-token={}", prefix, maxKeys, continuationToken);
11-
} else {
12-
url = baseUrl + std::format("?list-type=2&prefix={}&max-keys={}", prefix, maxKeys);
13-
}
7+
8+
// Build URL with query parameters
9+
std::string url = std::format("{}?list-type=2", buildURL(bucket));
10+
11+
if (options.Prefix.has_value())
12+
url += std::format("&prefix={}", options.Prefix.value());
13+
14+
int maxKeys = options.MaxKeys.value_or(1000);
15+
url += std::format("&max-keys={}", maxKeys);
16+
17+
if (options.ContinuationToken.has_value())
18+
url += std::format("&continuation-token={}", options.ContinuationToken.value());
19+
if (options.Delimiter.has_value())
20+
url += std::format("&delimiter={}", options.Delimiter.value());
21+
if (options.EncodingType.has_value())
22+
url += std::format("&encoding-type={}", options.EncodingType.value());
23+
if (options.StartAfter.has_value())
24+
url += std::format("&start-after={}", options.StartAfter.value());
25+
if (options.FetchOwner.has_value() && options.FetchOwner.value())
26+
url += "&fetch-owner=true";
1427

1528
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
29+
30+
// opt headers
31+
if (options.ExpectedBucketOwner.has_value())
32+
req.header("x-amz-expected-bucket-owner", options.ExpectedBucketOwner.value());
33+
34+
if (options.RequestPayer.has_value())
35+
req.header("x-amz-request-payer", options.RequestPayer.value());
36+
1637
Signer.sign(req);
1738
HttpResponse res = req.execute();
1839

1940
const std::vector<XMLNode>& XMLBody = Parser.parse(res.body());
2041

21-
if (res.status() != 200) {
22-
return std::unexpected<Error>(deserializeError(XMLBody));
42+
if (res.status() >= 200 && res.status() < 300) {
43+
return deserializeListBucketResult(XMLBody, maxKeys);
2344
}
24-
return deserializeListBucketResult(XMLBody, maxKeys);
45+
return std::unexpected<Error>(deserializeError(XMLBody));
2546
}
2647

2748
std::expected<ListObjectsResult, Error> S3Client::deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys) {
@@ -103,6 +124,8 @@ std::expected<ListObjectsResult, Error> S3Client::deserializeListBucketResult(co
103124
response.Contents[contentsIdx].Size = Parser.parseNumber<long>(node.value);
104125
} else if (node.tag == "ListBucketResult.Contents.StorageClass") {
105126
response.Contents[contentsIdx].StorageClass = std::move(node.value);
127+
} else if (node.tag == "ListBucketResult.CommonPrefixes.Prefix") {
128+
response.CommonPrefixes[commonPrefixesIdx].Prefix = std::move(node.value);
106129
} else {
107130
// Detect and parse error
108131
// Note(cristian): This fallback should not be needed as we have
@@ -132,20 +155,32 @@ std::expected<ListObjectsResult, Error> S3Client::deserializeListBucketResult(co
132155
return response;
133156
}
134157

135-
std::expected<std::string, Error> S3Client::GetObject(const std::string& bucket, const std::string& key) {
158+
std::expected<std::string, Error> S3Client::GetObject(const std::string& bucket, const std::string& key, const GetObjectInput& options) {
136159
std::string url = buildURL(bucket) + std::format("/{}", key);
137160

138161
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
162+
163+
// opt headers
164+
if (options.Range.has_value())
165+
req.header("Range", options.Range.value());
166+
if (options.If_Match.has_value())
167+
req.header("If-Match", options.If_Match.value());
168+
if (options.If_None_Match.has_value())
169+
req.header("If-None-Match", options.If_None_Match.value());
170+
if (options.If_Modified_Since.has_value())
171+
req.header("If-Modified-Since", options.If_Modified_Since.value());
172+
if (options.If_Unmodified_Since.has_value())
173+
req.header("If-Unmodified-Since", options.If_Unmodified_Since.value());
174+
139175
Signer.sign(req);
140176
HttpResponse res = req.execute();
141177

142178
const std::vector<XMLNode>& XMLBody = Parser.parse(res.body());
143179

144-
if (res.status() != 200) {
145-
return std::unexpected<Error>(deserializeError(XMLBody));
180+
if (res.status() >= 200 && res.status() < 300) {
181+
return res.body();
146182
}
147-
148-
return res.body();
183+
return std::unexpected<Error>(deserializeError(XMLBody));
149184
}
150185

151186
Error S3Client::deserializeError(const std::vector<XMLNode>& nodes) {

‎src/s3cpp/s3.h‎

Lines changed: 11 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,8 @@
11
#include <expected>
2-
#include <print>
32
#include <s3cpp/auth.h>
3+
#include <s3cpp/types.h>
44
#include <s3cpp/xml.hpp>
55

6-
// ListObjects
7-
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_ResponseSyntax
8-
9-
struct Contents_ {
10-
std::string ChecksumAlgorithm;
11-
std::string ChecksumType;
12-
std::string ETag;
13-
std::string Key;
14-
std::string LastModified;
15-
struct Owner_ {
16-
std::string DisplayName;
17-
std::string ID;
18-
} Owner;
19-
struct RestoreStatus_ {
20-
bool IsRestoreInProgress;
21-
std::string RestoreExpiryDate;
22-
} RestoreStatus;
23-
int64_t Size;
24-
std::string StorageClass;
25-
};
26-
27-
struct CommonPrefix {
28-
std::string Prefix;
29-
};
30-
31-
struct GetObjectInput {
32-
std::optional<std::string> If_Match;
33-
std::optional<std::string> If_Modified_Since;
34-
std::optional<std::string> If_None_Match;
35-
std::optional<std::string> If_Unmodified_Since;
36-
std::optional<int> partNumber;
37-
std::optional<std::string> Range; // e.g. bytes=0-9
38-
std::optional<std::string> response_cache_control;
39-
std::optional<std::string> response_content_disposition;
40-
std::optional<std::string> response_content_encoding;
41-
std::optional<std::string> response_content_language;
42-
std::optional<std::string> response_content_type;
43-
std::optional<std::string> response_expires;
44-
std::optional<std::string> versionId;
45-
};
46-
47-
struct ListObjectsResult {
48-
bool IsTruncated;
49-
std::string Marker;
50-
std::string NextMarker;
51-
std::vector<Contents_> Contents;
52-
std::string Name;
53-
std::string Prefix;
54-
std::string Delimiter;
55-
int MaxKeys;
56-
std::vector<CommonPrefix> CommonPrefixes;
57-
std::string EncodingType;
58-
int KeyCount;
59-
std::string ContinuationToken;
60-
std::string NextContinuationToken;
61-
std::string StartAfter;
62-
};
63-
64-
// REST generic error
65-
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses
66-
struct Error {
67-
std::string Code;
68-
std::string Message;
69-
std::string Resource;
70-
int RequestId;
71-
};
72-
73-
enum class S3AddressingStyle {
74-
VirtualHosted,
75-
PathStyle
76-
};
77-
786
class S3Client {
797
public:
808
// TODO(cristian): We should accept and define the endpoint url here
@@ -103,26 +31,9 @@ class S3Client {
10331
}
10432

10533
// S3 operations
106-
// TODO(cristian): ListObjectsV2 missing URI params:
107-
// - Bucket
108-
// - Continuation token
109-
// - Delimiter
110-
// - EncodingType
111-
// - ExpectedBucketOwner
112-
// - FetchOwner
113-
// - MaxKeys
114-
// - OptionalObjectAttributes
115-
// - Prefix
116-
// - RequestPayer
117-
// - StartAfter
118-
std::expected<ListObjectsResult, Error> ListObjects(const std::string& bucket) { return ListObjects(bucket, "/", 1000, ""); }
119-
std::expected<ListObjectsResult, Error> ListObjects(const std::string& bucket, const std::string& prefix) { return ListObjects(bucket, prefix, 1000, ""); }
120-
std::expected<ListObjectsResult, Error> ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys) { return ListObjects(bucket, prefix, maxKeys, ""); }
121-
std::expected<ListObjectsResult, Error> ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken);
34+
std::expected<ListObjectsResult, Error> ListObjects(const std::string& bucket, const ListObjectsInput& options = {});
12235

123-
std::expected<std::string, Error> GetObject(const std::string& bucket, const std::string& key);
124-
std::expected<std::string, Error> GetObject(const std::string& bucket, const std::string& key, const GetObjectInput& opt);
125-
// TODO(cristian): Add all overloading needed for different params
36+
std::expected<std::string, Error> GetObject(const std::string& bucket, const std::string& key, const GetObjectInput& options = {});
12637
// TODO(cristian): HeadBucket and HeadObject, PutObject, CreateBucket
12738

12839
// S3 responses
@@ -171,7 +82,14 @@ class ListObjectsPaginator {
17182
bool HasMorePages() const { return hasMorePages_; }
17283

17384
std::expected<ListObjectsResult, Error> NextPage() {
174-
auto response = client_.ListObjects(bucket_, prefix_, maxKeys_, continuationToken_);
85+
ListObjectsInput options;
86+
if (!continuationToken_.empty())
87+
options.ContinuationToken = continuationToken_;
88+
if (!prefix_.empty())
89+
options.Prefix = prefix_;
90+
options.MaxKeys = maxKeys_;
91+
92+
auto response = client_.ListObjects(bucket_, options);
17593
if (response.has_value()) {
17694
hasMorePages_ = response.value().IsTruncated;
17795
continuationToken_ = response.value().NextContinuationToken;

‎src/s3cpp/types.h‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#include <optional>
2+
#include <string>
3+
#include <vector>
4+
5+
enum class S3AddressingStyle {
6+
VirtualHosted,
7+
PathStyle
8+
};
9+
10+
// ListObjects
11+
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_ResponseSyntax
12+
struct ListObjectsInput {
13+
std::optional<std::string> ContinuationToken;
14+
std::optional<std::string> Delimiter;
15+
std::optional<std::string> EncodingType;
16+
std::optional<std::string> ExpectedBucketOwner;
17+
std::optional<bool> FetchOwner;
18+
std::optional<int> MaxKeys;
19+
std::optional<std::string> Prefix;
20+
std::optional<std::string> RequestPayer;
21+
std::optional<std::string> StartAfter;
22+
};
23+
24+
struct Contents_ {
25+
std::string ChecksumAlgorithm;
26+
std::string ChecksumType;
27+
std::string ETag;
28+
std::string Key;
29+
std::string LastModified;
30+
struct Owner_ {
31+
std::string DisplayName;
32+
std::string ID;
33+
} Owner;
34+
struct RestoreStatus_ {
35+
bool IsRestoreInProgress;
36+
std::string RestoreExpiryDate;
37+
} RestoreStatus;
38+
int64_t Size;
39+
std::string StorageClass;
40+
};
41+
42+
struct CommonPrefix {
43+
std::string Prefix;
44+
};
45+
46+
struct GetObjectInput {
47+
std::optional<std::string> If_Match;
48+
std::optional<std::string> If_Modified_Since;
49+
std::optional<std::string> If_None_Match;
50+
std::optional<std::string> If_Unmodified_Since;
51+
std::optional<int> partNumber;
52+
std::optional<std::string> Range; // e.g. bytes=0-9
53+
std::optional<std::string> response_cache_control;
54+
std::optional<std::string> response_content_disposition;
55+
std::optional<std::string> response_content_encoding;
56+
std::optional<std::string> response_content_language;
57+
std::optional<std::string> response_content_type;
58+
std::optional<std::string> response_expires;
59+
std::optional<std::string> versionId;
60+
};
61+
62+
struct ListObjectsResult {
63+
bool IsTruncated;
64+
std::string Marker;
65+
std::string NextMarker;
66+
std::vector<Contents_> Contents;
67+
std::string Name;
68+
std::string Prefix;
69+
std::string Delimiter;
70+
int MaxKeys;
71+
std::vector<CommonPrefix> CommonPrefixes;
72+
std::string EncodingType;
73+
int KeyCount;
74+
std::string ContinuationToken;
75+
std::string NextContinuationToken;
76+
std::string StartAfter;
77+
};
78+
79+
// REST generic error
80+
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses
81+
struct Error {
82+
std::string Code;
83+
std::string Message;
84+
std::string Resource;
85+
int RequestId;
86+
};

0 commit comments

Comments
 (0)