From b02bdf2516a025e05048faa8badf6407a7218aab Mon Sep 17 00:00:00 2001 From: Nick Morgan Date: Mon, 3 Jul 2023 20:29:39 -0400 Subject: [PATCH 1/4] Add handlers for s3 URL generation --- cmd/server/main.go | 3 ++ config/config.go | 12 +++-- go.mod | 18 +++++++ go.sum | 42 ++++++++++++++++ handlers/backup/get_s3_url.go | 65 +++++++++++++++++++++++++ handlers/handlers.go | 4 ++ s3/repository.go | 92 +++++++++++++++++++++++++++++++++++ 7 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 handlers/backup/get_s3_url.go create mode 100644 s3/repository.go diff --git a/cmd/server/main.go b/cmd/server/main.go index cdceb7a..d1ff9d1 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -25,6 +25,7 @@ import ( "github.com/coinbase-samples/waas-proxy-go/config" "github.com/coinbase-samples/waas-proxy-go/handlers" + "github.com/coinbase-samples/waas-proxy-go/s3" "github.com/coinbase-samples/waas-proxy-go/waas" ghandlers "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -53,6 +54,8 @@ func main() { log.Fatalf("unable to init WaaS clients: %v", err) } + s3.InitRepo(context.Background(), &app) + router := mux.NewRouter() handlers.RegisterHandlers(app, router) diff --git a/config/config.go b/config/config.go index 66ebc6b..3c7fb56 100644 --- a/config/config.go +++ b/config/config.go @@ -21,11 +21,13 @@ import ( ) type AppConfig struct { - Env string `mapstructure:"ENV_NAME"` - LogLevel string `mapstructure:"LOG_LEVEL"` - ApiKeyName string `mapstructure:"COINBASE_CLOUD_API_KEY_NAME"` - ApiPrivateKey string `mapstructure:"COINBASE_CLOUD_API_PRIVATE_KEY"` - AppUrl string `mapstructure:"APP_URL"` + Env string `mapstructure:"ENV_NAME"` + LogLevel string `mapstructure:"LOG_LEVEL"` + ApiKeyName string `mapstructure:"COINBASE_CLOUD_API_KEY_NAME"` + ApiPrivateKey string `mapstructure:"COINBASE_CLOUD_API_PRIVATE_KEY"` + AppUrl string `mapstructure:"APP_URL"` + BucketName string `mapstructure:"BUCKET_NAME"` + PresignedUrlExpiration int64 `mapstructure:"PRESIGNED_URL_EXPIRATION"` } func (a AppConfig) IsLocalEnv() bool { diff --git a/go.mod b/go.mod index 8ea45c0..3448c40 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,24 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect diff --git a/go.sum b/go.sum index 53adff2..c2c37a9 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,43 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= +github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= +github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= +github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -126,6 +163,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -166,6 +204,8 @@ github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -188,6 +228,7 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= @@ -535,6 +576,7 @@ gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/backup/get_s3_url.go b/handlers/backup/get_s3_url.go new file mode 100644 index 0000000..4379288 --- /dev/null +++ b/handlers/backup/get_s3_url.go @@ -0,0 +1,65 @@ +/** + * Copyright 2023 Coinbase Global, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package backup + +import ( + "net/http" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/coinbase-samples/waas-proxy-go/s3" + "github.com/coinbase-samples/waas-proxy-go/utils" + log "github.com/sirupsen/logrus" +) + +type GetPresignedS3UrlResponse struct { + Url string `json:"url"` +} + +func GetPresignedS3Url(w http.ResponseWriter, r *http.Request) { + + s3Method := utils.HttpPathVarOrSendBadRequest(w, r, "s3Method") + if len(s3Method) == 0 { + return + } + + objectKey := utils.HttpPathVarOrSendBadRequest(w, r, "objectKey") + if len(objectKey) == 0 { + return + } + + ctx := r.Context() + var httpReq *v4.PresignedHTTPRequest + var err error + if s3Method == "putObject" { + httpReq, err = s3.GeneratePutObjectUrl(ctx, objectKey) + } else { + httpReq, err = s3.GenerateGetObjectUrl(ctx, objectKey) + } + if err != nil { + utils.HttpBadGateway(w) + log.Error(err) + return + } + + resp := &GetPresignedS3UrlResponse{ + Url: httpReq.URL, + } + log.Debugf("GetPresignedS3Url response: %v", resp) + if err := utils.HttpMarshalAndWriteJsonResponseWithOk(w, resp); err != nil { + log.Errorf("cannot marshal and write GetPresignedS3UrlResponse: %v", err) + utils.HttpBadGateway(w) + } +} diff --git a/handlers/handlers.go b/handlers/handlers.go index 43a3696..e389a89 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -20,6 +20,7 @@ import ( "net/http" "github.com/coinbase-samples/waas-proxy-go/config" + "github.com/coinbase-samples/waas-proxy-go/handlers/backup" "github.com/coinbase-samples/waas-proxy-go/handlers/blockchain" "github.com/coinbase-samples/waas-proxy-go/handlers/combined" "github.com/coinbase-samples/waas-proxy-go/handlers/mpc_key" @@ -81,6 +82,9 @@ func RegisterHandlers(config config.AppConfig, router *mux.Router) { // Combined router.HandleFunc("/v1/waas/proxy/combined/pools/{poolId}/deviceGroups/{deviceGroupId}/mpcKeys/{mpcKeyId}/constructAndSign", combined.ConstructAndSign).Methods(http.MethodPost) router.HandleFunc("/v1/waas/proxy/combined/waitAndBroadcast", combined.WaitSignAndBroadcast).Methods(http.MethodPost) + + // S3 Urls + router.HandleFunc("/v1/waas/proxy/backup/method/{s3Method}/objectKey/{objectKey}", backup.GetPresignedS3Url).Methods(http.MethodGet) } func registerDefaultHandlers(config config.AppConfig, router *mux.Router) { diff --git a/s3/repository.go b/s3/repository.go new file mode 100644 index 0000000..2c8d1d4 --- /dev/null +++ b/s3/repository.go @@ -0,0 +1,92 @@ +/** + * Copyright 2023 Coinbase Global, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package s3 + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/coinbase-samples/waas-proxy-go/config" + log "github.com/sirupsen/logrus" +) + +var repo *Repository + +type Repository struct { + App *config.AppConfig + Client *s3.Client +} + +func InitRepo(ctx context.Context, a *config.AppConfig) { + cfg, err := awsConfig.LoadDefaultConfig(ctx) + + if err != nil { + log.Fatalf("unable to get aws config: %v", err) + } + + repo = &Repository{ + App: a, + Client: s3.NewFromConfig(cfg), + } +} + +func GenerateGetObjectUrl( + ctx context.Context, + objectKey string, +) (*v4.PresignedHTTPRequest, error) { + presignClient := s3.NewPresignClient(repo.Client) + request, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(repo.App.BucketName), + Key: aws.String(objectKey), + }, func(opts *s3.PresignOptions) { + opts.Expires = time.Duration(repo.App.PresignedUrlExpiration * int64(time.Second)) + }) + if err != nil { + return nil, fmt.Errorf( + "generate presigned get object url failed for object: %s err: %w", + objectKey, + err, + ) + } + return request, nil +} + +func GeneratePutObjectUrl( + ctx context.Context, + objectKey string, +) (*v4.PresignedHTTPRequest, error) { + presignClient := s3.NewPresignClient(repo.Client) + request, err := presignClient.PresignPutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(repo.App.BucketName), + ContentType: aws.String("application/octet-stream"), + Key: aws.String(objectKey), + }, func(opts *s3.PresignOptions) { + opts.Expires = time.Duration(repo.App.PresignedUrlExpiration * int64(time.Second)) + }) + if err != nil { + return nil, fmt.Errorf( + "generate presigned put object url failed for object: %s err: %w", + objectKey, + err, + ) + } + return request, nil +} From faa98e6aa79b66aff857748b6a473c90bd8db2bb Mon Sep 17 00:00:00 2001 From: Nick Morgan Date: Mon, 10 Jul 2023 11:25:10 -0400 Subject: [PATCH 2/4] Add flag to enable or disable S3 --- cmd/server/main.go | 4 +++- config/config.go | 1 + handlers/backup/get_s3_url.go | 2 +- handlers/handlers.go | 6 ++++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index d1ff9d1..76bf929 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -54,7 +54,9 @@ func main() { log.Fatalf("unable to init WaaS clients: %v", err) } - s3.InitRepo(context.Background(), &app) + if app.S3Enabled { + s3.InitRepo(context.Background(), &app) + } router := mux.NewRouter() diff --git a/config/config.go b/config/config.go index 3c7fb56..c408ffd 100644 --- a/config/config.go +++ b/config/config.go @@ -26,6 +26,7 @@ type AppConfig struct { ApiKeyName string `mapstructure:"COINBASE_CLOUD_API_KEY_NAME"` ApiPrivateKey string `mapstructure:"COINBASE_CLOUD_API_PRIVATE_KEY"` AppUrl string `mapstructure:"APP_URL"` + S3Enabled bool `mapstructure:"S3_ENABLED"` BucketName string `mapstructure:"BUCKET_NAME"` PresignedUrlExpiration int64 `mapstructure:"PRESIGNED_URL_EXPIRATION"` } diff --git a/handlers/backup/get_s3_url.go b/handlers/backup/get_s3_url.go index 4379288..51c847c 100644 --- a/handlers/backup/get_s3_url.go +++ b/handlers/backup/get_s3_url.go @@ -57,7 +57,7 @@ func GetPresignedS3Url(w http.ResponseWriter, r *http.Request) { resp := &GetPresignedS3UrlResponse{ Url: httpReq.URL, } - log.Debugf("GetPresignedS3Url response: %v", resp) + if err := utils.HttpMarshalAndWriteJsonResponseWithOk(w, resp); err != nil { log.Errorf("cannot marshal and write GetPresignedS3UrlResponse: %v", err) utils.HttpBadGateway(w) diff --git a/handlers/handlers.go b/handlers/handlers.go index e389a89..f5c6310 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -83,8 +83,10 @@ func RegisterHandlers(config config.AppConfig, router *mux.Router) { router.HandleFunc("/v1/waas/proxy/combined/pools/{poolId}/deviceGroups/{deviceGroupId}/mpcKeys/{mpcKeyId}/constructAndSign", combined.ConstructAndSign).Methods(http.MethodPost) router.HandleFunc("/v1/waas/proxy/combined/waitAndBroadcast", combined.WaitSignAndBroadcast).Methods(http.MethodPost) - // S3 Urls - router.HandleFunc("/v1/waas/proxy/backup/method/{s3Method}/objectKey/{objectKey}", backup.GetPresignedS3Url).Methods(http.MethodGet) + if config.S3Enabled { + // S3 Urls + router.HandleFunc("/v1/waas/proxy/backup/method/{s3Method}/objectKey/{objectKey}", backup.GetPresignedS3Url).Methods(http.MethodGet) + } } func registerDefaultHandlers(config config.AppConfig, router *mux.Router) { From ecfbce9011fc2ba9c35e5b185167a18351971cd5 Mon Sep 17 00:00:00 2001 From: Nick Morgan Date: Thu, 20 Jul 2023 14:51:38 -0400 Subject: [PATCH 3/4] update environment variable and directory structure --- {s3 => cloud/aws/s3}/repository.go | 0 cmd/server/main.go | 4 ++-- config/config.go | 2 +- go.mod | 8 ++++---- go.sum | 2 -- handlers/backup/get_s3_url.go | 2 +- handlers/handlers.go | 2 +- 7 files changed, 9 insertions(+), 11 deletions(-) rename {s3 => cloud/aws/s3}/repository.go (100%) diff --git a/s3/repository.go b/cloud/aws/s3/repository.go similarity index 100% rename from s3/repository.go rename to cloud/aws/s3/repository.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 76bf929..2587744 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -23,9 +23,9 @@ import ( "os/signal" "time" + "github.com/coinbase-samples/waas-proxy-go/cloud/aws/s3" "github.com/coinbase-samples/waas-proxy-go/config" "github.com/coinbase-samples/waas-proxy-go/handlers" - "github.com/coinbase-samples/waas-proxy-go/s3" "github.com/coinbase-samples/waas-proxy-go/waas" ghandlers "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -54,7 +54,7 @@ func main() { log.Fatalf("unable to init WaaS clients: %v", err) } - if app.S3Enabled { + if app.AwsS3Enabled { s3.InitRepo(context.Background(), &app) } diff --git a/config/config.go b/config/config.go index c408ffd..ef8d4ee 100644 --- a/config/config.go +++ b/config/config.go @@ -26,7 +26,7 @@ type AppConfig struct { ApiKeyName string `mapstructure:"COINBASE_CLOUD_API_KEY_NAME"` ApiPrivateKey string `mapstructure:"COINBASE_CLOUD_API_PRIVATE_KEY"` AppUrl string `mapstructure:"APP_URL"` - S3Enabled bool `mapstructure:"S3_ENABLED"` + AwsS3Enabled bool `mapstructure:"AWS_S3_ENABLED"` BucketName string `mapstructure:"BUCKET_NAME"` PresignedUrlExpiration int64 `mapstructure:"PRESIGNED_URL_EXPIRATION"` } diff --git a/go.mod b/go.mod index 3448c40..d12a37d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/coinbase-samples/waas-proxy-go go 1.19 require ( + github.com/aws/aws-sdk-go-v2 v1.18.1 + github.com/aws/aws-sdk-go-v2/config v1.18.27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 github.com/coinbase/waas-client-library-go v1.0.2 github.com/ethereum/go-ethereum v1.11.5 github.com/google/uuid v1.3.0 @@ -28,13 +31,11 @@ require ( golang.org/x/oauth2 v0.6.0 // indirect google.golang.org/api v0.114.0 google.golang.org/appengine v1.6.7 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 + gopkg.in/square/go-jose.v2 v2.6.0 // indirect ) require ( - github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect @@ -45,7 +46,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect diff --git a/go.sum b/go.sum index c2c37a9..a771b13 100644 --- a/go.sum +++ b/go.sum @@ -46,7 +46,6 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= @@ -228,7 +227,6 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= diff --git a/handlers/backup/get_s3_url.go b/handlers/backup/get_s3_url.go index 51c847c..6bc2cda 100644 --- a/handlers/backup/get_s3_url.go +++ b/handlers/backup/get_s3_url.go @@ -19,7 +19,7 @@ import ( "net/http" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/coinbase-samples/waas-proxy-go/s3" + "github.com/coinbase-samples/waas-proxy-go/cloud/aws/s3" "github.com/coinbase-samples/waas-proxy-go/utils" log "github.com/sirupsen/logrus" ) diff --git a/handlers/handlers.go b/handlers/handlers.go index f5c6310..4987eab 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -83,7 +83,7 @@ func RegisterHandlers(config config.AppConfig, router *mux.Router) { router.HandleFunc("/v1/waas/proxy/combined/pools/{poolId}/deviceGroups/{deviceGroupId}/mpcKeys/{mpcKeyId}/constructAndSign", combined.ConstructAndSign).Methods(http.MethodPost) router.HandleFunc("/v1/waas/proxy/combined/waitAndBroadcast", combined.WaitSignAndBroadcast).Methods(http.MethodPost) - if config.S3Enabled { + if config.AwsS3Enabled { // S3 Urls router.HandleFunc("/v1/waas/proxy/backup/method/{s3Method}/objectKey/{objectKey}", backup.GetPresignedS3Url).Methods(http.MethodGet) } From d130a1eca70b49b517b8f218ca41f4a59b1926a0 Mon Sep 17 00:00:00 2001 From: Nick Morgan Date: Mon, 24 Jul 2023 10:45:21 -0400 Subject: [PATCH 4/4] update env variable to clarify units --- cloud/aws/s3/repository.go | 4 ++-- config/config.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cloud/aws/s3/repository.go b/cloud/aws/s3/repository.go index 2c8d1d4..2e12beb 100644 --- a/cloud/aws/s3/repository.go +++ b/cloud/aws/s3/repository.go @@ -57,7 +57,7 @@ func GenerateGetObjectUrl( Bucket: aws.String(repo.App.BucketName), Key: aws.String(objectKey), }, func(opts *s3.PresignOptions) { - opts.Expires = time.Duration(repo.App.PresignedUrlExpiration * int64(time.Second)) + opts.Expires = time.Duration(repo.App.PresignedUrlExpirationInSec * int64(time.Second)) }) if err != nil { return nil, fmt.Errorf( @@ -79,7 +79,7 @@ func GeneratePutObjectUrl( ContentType: aws.String("application/octet-stream"), Key: aws.String(objectKey), }, func(opts *s3.PresignOptions) { - opts.Expires = time.Duration(repo.App.PresignedUrlExpiration * int64(time.Second)) + opts.Expires = time.Duration(repo.App.PresignedUrlExpirationInSec * int64(time.Second)) }) if err != nil { return nil, fmt.Errorf( diff --git a/config/config.go b/config/config.go index ef8d4ee..68eb79c 100644 --- a/config/config.go +++ b/config/config.go @@ -21,14 +21,14 @@ import ( ) type AppConfig struct { - Env string `mapstructure:"ENV_NAME"` - LogLevel string `mapstructure:"LOG_LEVEL"` - ApiKeyName string `mapstructure:"COINBASE_CLOUD_API_KEY_NAME"` - ApiPrivateKey string `mapstructure:"COINBASE_CLOUD_API_PRIVATE_KEY"` - AppUrl string `mapstructure:"APP_URL"` - AwsS3Enabled bool `mapstructure:"AWS_S3_ENABLED"` - BucketName string `mapstructure:"BUCKET_NAME"` - PresignedUrlExpiration int64 `mapstructure:"PRESIGNED_URL_EXPIRATION"` + Env string `mapstructure:"ENV_NAME"` + LogLevel string `mapstructure:"LOG_LEVEL"` + ApiKeyName string `mapstructure:"COINBASE_CLOUD_API_KEY_NAME"` + ApiPrivateKey string `mapstructure:"COINBASE_CLOUD_API_PRIVATE_KEY"` + AppUrl string `mapstructure:"APP_URL"` + AwsS3Enabled bool `mapstructure:"AWS_S3_ENABLED"` + BucketName string `mapstructure:"BUCKET_NAME"` + PresignedUrlExpirationInSec int64 `mapstructure:"PRESIGNED_URL_EXPIRATION_SECONDS"` } func (a AppConfig) IsLocalEnv() bool {