From da835efe256ba0875c70e49457d8443f00a402de Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 19 Nov 2018 17:41:36 -0800 Subject: [PATCH 01/15] ORGANIC-443. Disabled lint that stops the build. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bdf5ffb..6109281 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CHECK_FILES?=$$(go list ./... | grep -v /vendor/) help: ## Show this help. @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) -all: lint vet test build ## Run the tests and build the binary. +all: vet test build ## Run the tests and build the binary. build: ## Build the binary. go build -ldflags "-X github.com/netlify/git-gateway/cmd.Version=`git rev-parse HEAD`" From 46b42ae11fcc267a6568470a3b5d1a401d4dfc1f Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 19 Nov 2018 17:42:16 -0800 Subject: [PATCH 02/15] ORGANIC-443. Updated Dockerfile for new golang version. (needed for Okta lib). --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 480c7cb..d5a8859 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ -FROM netlify/go-glide:v0.12.3 +FROM golang:1.11.2-stretch ADD . /go/src/github.com/netlify/git-gateway +RUN go get github.com/Masterminds/glide + RUN useradd -m netlify && cd /go/src/github.com/netlify/git-gateway && make deps build && mv git-gateway /usr/local/bin/ USER netlify From 151ec09081cfdee699748e44a4a4aa0ffaf0c683 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 19 Nov 2018 18:15:19 -0800 Subject: [PATCH 03/15] ORGANIC-443. !!Hacks introduced (not for production)!! Reimplemented jwt validation with Okta's golang lib. Made a few hacks to get it working. --- api/auth.go | 33 ++++++++++--- api/github.go | 9 +++- conf/configuration.go | 3 ++ glide.lock | 111 ++++++++++++++++++++++++++---------------- glide.yaml | 2 + 5 files changed, 108 insertions(+), 50 deletions(-) diff --git a/api/auth.go b/api/auth.go index d1b380a..888a7e5 100644 --- a/api/auth.go +++ b/api/auth.go @@ -4,8 +4,8 @@ import ( "context" "net/http" - jwt "github.com/dgrijalva/jwt-go" "github.com/sirupsen/logrus" + "github.com/okta/okta-jwt-verifier-golang" ) // requireAuthentication checks incoming requests for tokens presented using the Authorization header @@ -35,14 +35,35 @@ func (a *API) extractBearerToken(w http.ResponseWriter, r *http.Request) (string } func (a *API) parseJWTClaims(bearer string, r *http.Request) (context.Context, error) { + // Reimplemented to use Okta lib + // Original validation only work for HS256 algo, + // Okta supports RS256 only which requires public key downloading and caching (key rotation) config := getConfig(r.Context()) - p := jwt.Parser{ValidMethods: []string{jwt.SigningMethodHS256.Name}} - token, err := p.ParseWithClaims(bearer, &GatewayClaims{}, func(token *jwt.Token) (interface{}, error) { - return []byte(config.JWT.Secret), nil - }) + + toValidate := map[string]string{} + toValidate["aud"] = config.JWT.AUD + toValidate["cid"] = config.JWT.CID + + jwtVerifierSetup := jwtverifier.JwtVerifier{ + Issuer: config.JWT.Issuer, + ClaimsToValidate: toValidate, + } + + verifier := jwtVerifierSetup.New() + + _, err := verifier.VerifyAccessToken(bearer) + + // @TODO? WARNING: Should be roles and other claims be checked here? + if err != nil { return nil, unauthorizedError("Invalid token: %v", err) } - return withToken(r.Context(), token), nil + logrus.Infof("parseJWTClaims passed") + + // return nil, because the `github.go` is coded to send personal token + // both github oauth generates its own id, so oauth pass-thru is impossible + // we can improve the gateway to talk oauth with github.com, but we will + // still return nil here. + return nil, nil } diff --git a/api/github.go b/api/github.go index 72ff074..8514017 100644 --- a/api/github.go +++ b/api/github.go @@ -74,6 +74,9 @@ func (gh *GitHubGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { } ctx = withProxyTarget(ctx, target) ctx = withAccessToken(ctx, config.GitHub.AccessToken) + + log := getLogEntry(r) + log.Infof("proxy.ServeHTTP: %+v\n", r.WithContext(ctx)) gh.proxy.ServeHTTP(w, r.WithContext(ctx)) } @@ -82,8 +85,12 @@ func (gh *GitHubGateway) authenticate(w http.ResponseWriter, r *http.Request) er claims := getClaims(ctx) config := getConfig(ctx) + log := getLogEntry(r) + log.Infof("authenticate context: %v+", ctx) if claims == nil { - return errors.New("Access to endpoint not allowed: no claims found in Bearer token") + // @TODO? WARNING: the check should be done in auth.go, imo. + // Having the jwt in the context (and thus, sent to github.com) is not necessary + // return errors.New("Access to endpoint not allowed: no claims found in Bearer token") } if !allowedRegexp.MatchString(r.URL.Path) { diff --git a/conf/configuration.go b/conf/configuration.go index d162f8a..6c6b688 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -46,6 +46,9 @@ type DBConfiguration struct { // JWTConfiguration holds all the JWT related configuration. type JWTConfiguration struct { Secret string `json:"secret" required:"true"` + CID string `envconfig:"CLIENT_ID" json:"client_id,omitempty"` + Issuer string `envconfig:"ISSUER" json:"issuer,omitempty"` + AUD string `envconfig:"AUD" json:"aud,omitempty"` } // GlobalConfiguration holds all the configuration that applies to all instances. diff --git a/glide.lock b/glide.lock index 125fb36..7efa71b 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ -hash: 584c9db44ed61e38b043e033cc12c9a6f229a5a91a110b1f4a69bf48f843e0f1 -updated: 2017-08-21T11:29:52.316754668-04:00 +hash: d16d6e38e0df8d4315d0630917bff19bc885c18c3566e74638fa8c2fc59f2541 +updated: 2018-11-16T13:17:46.231317-08:00 imports: - name: cloud.google.com/go - version: 06f11fffc537c4aef126d9fd3a92e2d7968f118f + version: 74b12019e2aa53ec27882158f59192d7cd6d1998 subpackages: - compute/metadata - name: github.com/badoux/checkmail - version: d0a759655d62bcdc95c50a0676f3e9702ed59453 + version: 0755fe2dc241caebab64327c352006712f6a55c4 - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: @@ -14,7 +14,7 @@ imports: - name: github.com/dgrijalva/jwt-go version: d2709f9f1f31ebcda9651b03077758c1f3a0018c - name: github.com/fsnotify/fsnotify - version: 4da3e2cfbabc9f751898f250b49f2439785783a1 + version: ccc981bf80385c528a65fbfdd49bf2d8da22aa23 - name: github.com/go-chi/chi version: b8567b6442e27704bfeb725095091581cbe82a00 subpackages: @@ -22,7 +22,7 @@ imports: - name: github.com/go-sql-driver/mysql version: a0583e0143b1624142adab07e0e97fe106d99561 - name: github.com/golang/protobuf - version: ab9f9a6dab164b7d1246e0e688b0ab7b94d8553e + version: 52132540909e117f2b98b0694383dc0ab1e1deca subpackages: - proto - name: github.com/GoogleCloudPlatform/cloudsql-proxy @@ -30,14 +30,16 @@ imports: subpackages: - logging - proxy/certs + - proxy/dialers/mysql - proxy/dialers/postgres - proxy/proxy - proxy/util - name: github.com/hashicorp/hcl - version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca + version: 65a6292f0157eff210d03ed1bf6c59b190b8b906 subpackages: - hcl/ast - hcl/parser + - hcl/printer - hcl/scanner - hcl/strconv - hcl/token @@ -45,36 +47,54 @@ imports: - json/scanner - json/token - name: github.com/imdario/mergo - version: 3e95a51e0639b4cf372f2ccf74c86749d747fbdc + version: ca3dcc1022bae9b5510f3c83705b72db1b1a96f9 - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 - name: github.com/jinzhu/gorm version: 5b8c0dd6b92d9caa8036c31dcb117f2df7cceefa - name: github.com/jinzhu/inflection - version: 1c35d901db3da928c72a72d8458480cc9ade058f + version: 04140366298a54a039076d798123ffa108fff46c - name: github.com/joho/godotenv version: 726cc8b906e3d31c70a9671c90a13716a8d3f50d - name: github.com/kelseyhightower/envconfig version: f611eb38b3875cc3bd991ca91c51d06446afa14c - name: github.com/kr/logfmt version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 +- name: github.com/lestrrat-go/jwx + version: 0d477e6a1f0ec69634840308e3873fe71d928dd0 + subpackages: + - internal/base64 + - internal/option + - jwa + - jwk + - jws + - jws/sign + - jws/verify +- name: github.com/lestrrat-go/pdebug + version: 39f9a71bcabe9432cbdfe4d3d33f41988acd2ce6 - name: github.com/lib/pq - version: e42267488fe361b9dc034be7a6bffef5b195bceb + version: 9eb73efc1fcc404148b56765b0d3f61d9a5ef8ee subpackages: - oid - name: github.com/magiconair/properties - version: be5ece7dd465ab0765a9682137865547526d1dfb + version: c2353362d570a7bfa228149c62842019201cfb71 +- name: github.com/mailru/easyjson + version: 60711f1a8329503b04e1c88535f419d0bb440bff + subpackages: + - buffer + - jlexer + - jwriter - name: github.com/mattn/go-sqlite3 version: 3b3f1d01b2696af5501697c35629048c227586ab - name: github.com/mitchellh/mapstructure - version: d0303fe809921458f417bcf828397a65db30a7e4 + version: 3536a929edddb9a5b34bd6861dc4a9647cb459fe - name: github.com/nats-io/nats version: 61923ed1eaf8398000991fbbee2ef11ab5a5be0d subpackages: - encoders/builtin - util - name: github.com/nats-io/nuid - version: 3cf34f9fca4e88afa9da8eabd75e3326c9941b44 + version: 3024a71c3cbe30667286099921591e6fcc328230 - name: github.com/netlify/mailme version: c4a76ce443c1122ead2518b28cc7ffaf1091cc9a - name: github.com/netlify/netlify-commons @@ -86,16 +106,20 @@ imports: - metrics/transport - nconf - tls +- name: github.com/okta/okta-jwt-verifier-golang + version: 04702def3e1b9b1c6b419c9c3aae1ec184a5d4b2 + subpackages: + - adaptors + - adaptors/lestrratGoJwx + - discovery + - discovery/oidc + - errors - name: github.com/pborman/uuid version: a97ce2ca70fa5a848076093f05e639a89ca34d06 - name: github.com/pelletier/go-toml - version: 4692b8f9babfc93db58cc592ba2689d8736781de + version: aa79e12a973495d00b01f2353330e5275d084731 - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib - name: github.com/rs/cors version: 8dd4211afb5d08dbb39a533b9bb9e4b486351df6 - name: github.com/rybit/nats_logrus_hook @@ -107,7 +131,7 @@ imports: - name: github.com/signalfx/gohistogram version: 1ccfd2ff508314074672f4450a917011a2060408 - name: github.com/signalfx/golib - version: cb7680940d605b817db79790c241eed2a00fa6e6 + version: 33764ade36dd90eb1593000ebee93ac1f9d9d419 subpackages: - datapoint - errors @@ -115,25 +139,28 @@ imports: - eventcounter - log - sfxclient + - sfxclient/spanfilter - timekeeper + - trace + - trace/format - name: github.com/sirupsen/logrus version: a3f95b5c423586578a4e099b11a46c2479628cac - name: github.com/spf13/afero - version: 9be650865eab0c12963d8753212f4f9c66cdcf12 + version: d40851caa0d747393da1ffb28f7f9d8b4eeffebd subpackages: - mem - name: github.com/spf13/cast - version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 + version: 8c9545af88b134710ab1cd196795e7f2388358d7 - name: github.com/spf13/cobra - version: 4a7b7e65864c064d48dce82efbbfed2bdc0bf2aa + version: fe5e611709b0c57fa4a89136deaa8e1d4004d053 - name: github.com/spf13/jwalterweatherman - version: 0efa5202c04663c757d84f90f5219c1250baf94f + version: 94f6ae3ed3bceceafa716478c5fbf8d29ca601a1 - name: github.com/spf13/pflag - version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 + version: aea12ed6721610dc6ed40141676d7ab0a1dac9e9 - name: github.com/spf13/viper - version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 + version: ae103d7e593e371c69e832d5eb3347e2b80cbbc9 - name: github.com/streadway/amqp - version: 2cbfe40c9341ad63ba23e53013b3ddc7989d801c + version: 27835f1a64e97101d95306211f03c0620ffa295d - name: github.com/stretchr/testify version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: @@ -141,17 +168,16 @@ imports: - require - suite - name: golang.org/x/crypto - version: eb71ad9bd329b5ac0fd0148dd99bd62e8be8e035 + version: 3d3f9f413869b949e48070b5bc593aa22cc2b8f2 subpackages: - bcrypt - - blowfish - name: golang.org/x/net - version: 1c05540f6879653db88113bc4a2b70aec4bd491f + version: adae6a3d119ae4890b46832a2e88a95adc62b8e7 subpackages: - context - context/ctxhttp - name: golang.org/x/oauth2 - version: 9a379c6b3e95a790ffc43293c2a78dee0d7b6e20 + version: f42d05182288abf10faef86d16c0d07b8d40ea2d subpackages: - bitbucket - github @@ -160,23 +186,23 @@ imports: - jws - jwt - name: golang.org/x/sys - version: 07c182904dbd53199946ba614a412c61d3c548f5 + version: 93218def8b18e66adbdab3eca8ec334700329f1f subpackages: - unix - name: golang.org/x/text - version: e56139fd9c5bc7244c76116c68e500765bb6db6b + version: 6f44c5a2ea40ee3593d98cdcc905cc1fdaa660e2 subpackages: - transform - unicode/norm - name: google.golang.org/api - version: ed10e890a8366167a7ce33fac2b12447987bcb1c + version: 83a9d304b1e613fc253e1e2710778642fe81af53 subpackages: - gensupport - googleapi - googleapi/internal/uritemplates - sqladmin/v1beta4 - name: google.golang.org/appengine - version: d9a072cfa7b9736e44311ef77b3e09d804bfa599 + version: 4a4468ece617fc8205e99368fa2200e9d1fad421 subpackages: - internal - internal/app_identity @@ -187,22 +213,21 @@ imports: - internal/remote_api - internal/urlfetch - urlfetch -- name: gopkg.in/alexcesaro/quotedprintable.v3 - version: 2caba252f4dc53eaf6b553000885530023f54623 - name: gopkg.in/gomail.v2 version: 41f3572897373c5538c50a2402db15db079fa4fd - name: gopkg.in/logfmt.v0 version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 - name: gopkg.in/mgo.v2 - version: 3f83fa5005286a7fe593b055f0d7771a7dce4655 + version: 9856a29383ce1c59f308dd1cf0363a79b5bef6b5 subpackages: - bson - - internal/json - - internal/sasl - - internal/scram - txn - name: gopkg.in/stack.v1 - version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf + version: 2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a - name: gopkg.in/yaml.v2 - version: eb3733d160e74a9c7e442f435eb3bea458e1d19f -testImports: [] + version: 5420a8b6744d3b0345ab293f6fcba19c978f1183 +testImports: +- name: github.com/pmezard/go-difflib + version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + subpackages: + - difflib diff --git a/glide.yaml b/glide.yaml index 20f99fc..18d20d6 100644 --- a/glide.yaml +++ b/glide.yaml @@ -2,6 +2,8 @@ package: github.com/netlify/git-gateway import: - package: github.com/dgrijalva/jwt-go version: v3.0.0 +- package: github.com/okta/okta-jwt-verifier-golang + version: 04702def3e1b9b1c6b419c9c3aae1ec184a5d4b2 - package: github.com/jinzhu/gorm version: 5b8c0dd6b92d9caa8036c31dcb117f2df7cceefa - package: github.com/pborman/uuid From 60da7c10cb7fcd1601f7f81f8e6d1dd4381a4617 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 19 Nov 2018 18:18:10 -0800 Subject: [PATCH 04/15] ORGANIC-443. Fixed token key for REST to github.com. Confirmed github only work with `Authorization: token`, not `access_token` (would it broke gitlab, bitbucket?). --- api/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/context.go b/api/context.go index e8d2a8a..583d825 100644 --- a/api/context.go +++ b/api/context.go @@ -20,7 +20,7 @@ func (c contextKey) String() string { } const ( - accessTokenKey = contextKey("access_token") + accessTokenKey = contextKey("token") tokenKey = contextKey("jwt") requestIDKey = contextKey("request_id") configKey = contextKey("config") From fb04f6bcbf0288560970c9c10b9481e95444f831 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 19 Nov 2018 18:40:57 -0800 Subject: [PATCH 05/15] ORGANIC-443. Updated example.env that actual works. --- example.env | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/example.env b/example.env index 2d6691f..5a5ca04 100644 --- a/example.env +++ b/example.env @@ -1,12 +1,30 @@ -GITGATEWAY_JWT_SECRET="CHANGE-THIS! VERY IMPORTANT!" +# Warning: Many configuration would not work with quote (ie ""). + +# JWT Secret is not needed for RS256. Instead, issuer should be specified +# (eg, https://dev-1234.oktapreview.com/oauth2/default) +GITGATEWAY_JWT_SECRET= + +# @TODO - REQUIRED for Okta +ISSUER= + +# @TODO - REQUIRED for Okta +CLIENT_ID= + +AUD=api://default GITGATEWAY_DB_DRIVER=sqlite3 DATABASE_URL=gorm.db -GITGATEWAY_API_HOST=localhost -PORT=9999 +# @TODO - Is there way to expose internal port from Docker? +GITGATEWAY_API_HOST=0.0.0.0 +PORT=8087 + +# @TODO - REQUIRED +GITGATEWAY_GITHUB_ACCESS_TOKEN= -GITGATEWAY_GITHUB_ACCESS_TOKEN="personal-access-token" -GITGATEWAY_GITHUB_REPO="owner/name" +# @TODO - REQUIRED +GITGATEWAY_GITHUB_REPO= -GITGATEWAY_ROLES="admin,cms" # leave blank to allow all roles +# Original example.env wrote: leave blank to allow all roles. But, it won't +# work unless it is commented out +# GITGATEWAY_ROLES= From f643682aa8f0fdc8aeeec134879ddd20e1592227 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 19 Nov 2018 18:55:21 -0800 Subject: [PATCH 06/15] ORGANIC-443. Converted to use a smaller golang official image to speed up build. --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5a8859..1ecc22b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,14 @@ -FROM golang:1.11.2-stretch +FROM golang:1.11.2-alpine3.8 ADD . /go/src/github.com/netlify/git-gateway +RUN apk update && apk add wget && rm -rf /var/cache/apk/* + +RUN apk add --update alpine-sdk + RUN go get github.com/Masterminds/glide -RUN useradd -m netlify && cd /go/src/github.com/netlify/git-gateway && make deps build && mv git-gateway /usr/local/bin/ +RUN adduser -D -u 1000 netlify && cd /go/src/github.com/netlify/git-gateway && make deps build && mv git-gateway /usr/local/bin/ USER netlify CMD ["git-gateway"] From 5a7569d53821e14402812d5ca0b04e19091e5c69 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Tue, 20 Nov 2018 16:29:01 -0800 Subject: [PATCH 07/15] ORGANIC-443. Added instructions to run git-gateway. --- .gitignore | 1 + README.md | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ec26daa..cb13f5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +my.env gorm.db vendor/ git-gateway diff --git a/README.md b/README.md index 44341c1..da323d9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ However, for most use cases you won’t want to require all content editors to h Netlify’s Git Gateway lets you setup a gateway to your choice of Git provider's API ( now available with both GitHub and GitLab 🎉 ) that lets tools like Netlify CMS work with content, branches and pull requests on your users’ behalf. -The Git Gateway works with any identity service that can issue JWTs and only allows access when a JSON Web Token with sufficient permissions is present. +The Git Gateway works with some supported identity service that can issue JWTs and only allows access when a JSON Web Token with sufficient permissions is present. To configure the gateway, see our `example.env` file @@ -29,3 +29,32 @@ for GitLab: /repos/:owner/:name/commits/ /repos/:owner/:name/tree/ ``` + +**Running git-gateway to test locally** +**(Do not merge this section back to the open source project)** +**(Do not deploy it to production. It is a Proof of Concept has has not been secured. See @TODO items in code.** +**(the instruction assume Okta, and github.com)** + +1. pull down this project +2. generate a `personal access token` on github. (recommended: using a test account and w/ `repo:status` and `public_repo` permission only) + https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ +3. `cp example.env my.env` +4. update `GITGATEWAY_GITHUB_ACCESS_TOKEN` value in `my.env` accordingly +5. update `GITGATEWAY_GITHUB_REPO` value in `my.env` (it will be where the content being stored, eg, `owner/netlify-test`.) +6. sign up for a Dev account on Okta: https://developer.okta.com/signup/ +7. create a SPA Application onto the Dev account: + a. fill out the details + b. Pick "Send ID Token directly to app (Okta Simplified)" + c. have redirect uri points to the url of your content-cms ip:port + (eg, `http://localhost:8080/admin` etc, see, https://github.com/<< your org >>/content-cms) +8. update `ISSUER` value in `my.env` accordingly (eg, `https://dev-1234.oktapreview.com/oauth2/default`) +9. update `CLIENT_ID` value in `my.env` accordingly (eg, `32q897q234q324rq42322q`) +10. install Docker and add the `localdev` network +11. inspect Dockfile and then build the docker with this command: + `docker build -t netlify/git-gateway:latest .` +12. run `git-gateway` with this command: + `docker run --rm --env-file my.env --net localdev -p 127.0.0.1:8087:8087 --expose 8087 -ti --name netlify-git-gateway "netlify/git-gateway:latest"` +13. update `config.yml` in your content-cms repo (ie, https://github.com/<< your org >>/content-cms). + change `backend.name` value to `git-gateway` + change `backend.gateway_url` value to `http://localhost:8087` +14. run `content-cms` following the README.md From 914448deb09d5af65225a651eaaf53ff93c8c60c Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Wed, 21 Nov 2018 16:57:19 -0800 Subject: [PATCH 08/15] ORGANIC-467. Updated Dockerfile for build it slighlty faster. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ecc22b..f28c7f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ FROM golang:1.11.2-alpine3.8 ADD . /go/src/github.com/netlify/git-gateway -RUN apk update && apk add wget && rm -rf /var/cache/apk/* - RUN apk add --update alpine-sdk +RUN rm -rf /var/cache/apk/* + RUN go get github.com/Masterminds/glide RUN adduser -D -u 1000 netlify && cd /go/src/github.com/netlify/git-gateway && make deps build && mv git-gateway /usr/local/bin/ From e70ec3012a5150eda637e6283a9da9de622e47d8 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Mon, 26 Nov 2018 21:01:46 -0800 Subject: [PATCH 09/15] ORGANIC-467. Refactored authorization/authenication out of backends into auth.go. Got it compiled. --- README.md | 19 +++++++++++++++- api/api.go | 12 +++++----- api/auth.go | 56 +++++++++++++++++++++++++++++++++++++++++++---- api/bitbucket.go | 39 --------------------------------- api/github.go | 43 ------------------------------------ api/gitlab.go | 39 --------------------------------- api/middleware.go | 2 +- 7 files changed, 78 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index da323d9..e7d263b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ for GitLab: /repos/:owner/:name/tree/ ``` -**Running git-gateway to test locally** +**Running `git-gateway`** **(Do not merge this section back to the open source project)** **(Do not deploy it to production. It is a Proof of Concept has has not been secured. See @TODO items in code.** **(the instruction assume Okta, and github.com)** @@ -58,3 +58,20 @@ for GitLab: change `backend.name` value to `git-gateway` change `backend.gateway_url` value to `http://localhost:8087` 14. run `content-cms` following the README.md + +**Develop, Build and Run git-gateway** + +1. Follow instructions 1 - 10 in previous "Running `git-gateway`" section +2. Run these commands once: + ``` + docker build -t netlify/git-gateway:latest . + docker run --rm --env-file my.env --net localdev -p 127.0.0.1:8087:8087 --expose 8087 -ti -v $PWD:/go/src/github.com/netlify/git-gateway --entrypoint '/bin/sh' --user root netlify/git-gateway:latest + cd /go/src/github.com/netlify/git-gateway + make deps + ``` +3. Run these commands after edit: + ``` + make build && ./git-gateway + ``` +4. ` + c` to stop + diff --git a/api/api.go b/api/api.go index 99eed03..c980997 100644 --- a/api/api.go +++ b/api/api.go @@ -27,6 +27,7 @@ var bearerRegexp = regexp.MustCompile(`^(?:B|b)earer (\S+$)`) type API struct { handler http.Handler db storage.Connection + auth Auth config *conf.GlobalConfiguration version string } @@ -58,7 +59,8 @@ func NewAPI(globalConfig *conf.GlobalConfiguration, db storage.Connection) *API // NewAPIWithVersion creates a new REST API using the specified version func NewAPIWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, db storage.Connection, version string) *API { - api := &API{config: globalConfig, db: db, version: version} + auth := NewAuthWithVersion(ctx, globalConfig, version) + api := &API{config: globalConfig, db: db, auth: *auth, version: version} xffmw, _ := xff.Default() @@ -75,10 +77,10 @@ func NewAPIWithVersion(ctx context.Context, globalConfig *conf.GlobalConfigurati r.Use(api.loadJWSSignatureHeader) r.Use(api.loadInstanceConfig) } - r.With(api.requireAuthentication).Mount("/github", NewGitHubGateway()) - r.With(api.requireAuthentication).Mount("/gitlab", NewGitLabGateway()) - r.With(api.requireAuthentication).Mount("/bitbucket", NewBitBucketGateway()) - r.With(api.requireAuthentication).Get("/settings", api.Settings) + r.With(api.auth.authenticate).Mount("/github", NewGitHubGateway()) + r.With(api.auth.authenticate).Mount("/gitlab", NewGitLabGateway()) + r.With(api.auth.authenticate).Mount("/bitbucket", NewBitBucketGateway()) + r.With(api.auth.authenticate).Get("/settings", api.Settings) }) if globalConfig.MultiInstanceMode { diff --git a/api/auth.go b/api/auth.go index 888a7e5..bf00f30 100644 --- a/api/auth.go +++ b/api/auth.go @@ -2,14 +2,21 @@ package api import ( "context" + "errors" "net/http" + "github.com/netlify/git-gateway/conf" "github.com/sirupsen/logrus" "github.com/okta/okta-jwt-verifier-golang" ) -// requireAuthentication checks incoming requests for tokens presented using the Authorization header -func (a *API) requireAuthentication(w http.ResponseWriter, r *http.Request) (context.Context, error) { +type Auth struct { + config *conf.GlobalConfiguration + version string +} + +// authenicate checks incoming requests for tokens presented using the Authorization header +func (a *Auth) authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) { logrus.Info("Getting auth token") token, err := a.extractBearerToken(w, r) if err != nil { @@ -20,7 +27,48 @@ func (a *API) requireAuthentication(w http.ResponseWriter, r *http.Request) (con return a.parseJWTClaims(token, r) } -func (a *API) extractBearerToken(w http.ResponseWriter, r *http.Request) (string, error) { +// authorize checks incoming requests for roles data in tokens that is parsed and verified by prior authentication step +func (a *Auth) authorize(w http.ResponseWriter, r *http.Request) (context.Context, error) { + ctx := r.Context() + claims := getClaims(ctx) + config := getConfig(ctx) + + logrus.Infof("authenticate context: %v+", ctx) + if claims == nil { + return nil, errors.New("Access to endpoint not allowed: no claims found in Bearer token") + } + + if !allowedRegexp.MatchString(r.URL.Path) { + return nil, errors.New("Access to endpoint not allowed: this part of GitHub's API has been restricted") + } + + if len(config.Roles) == 0 { + return ctx, nil + } + + roles, ok := claims.AppMetaData["roles"] + if ok { + roleStrings, _ := roles.([]interface{}) + for _, data := range roleStrings { + role, _ := data.(string) + for _, adminRole := range config.Roles { + if role == adminRole { + return ctx, nil + } + } + } + } + + return nil, errors.New("Access to endpoint not allowed: your role doesn't allow access") +} + +func NewAuthWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, version string) *Auth { + auth := &Auth{config: globalConfig, version: version} + + return auth +} + +func (a *Auth) extractBearerToken(w http.ResponseWriter, r *http.Request) (string, error) { authHeader := r.Header.Get("Authorization") if authHeader == "" { return "", unauthorizedError("This endpoint requires a Bearer token") @@ -34,7 +82,7 @@ func (a *API) extractBearerToken(w http.ResponseWriter, r *http.Request) (string return matches[1], nil } -func (a *API) parseJWTClaims(bearer string, r *http.Request) (context.Context, error) { +func (a *Auth) parseJWTClaims(bearer string, r *http.Request) (context.Context, error) { // Reimplemented to use Okta lib // Original validation only work for HS256 algo, // Okta supports RS256 only which requires public key downloading and caching (key rotation) diff --git a/api/bitbucket.go b/api/bitbucket.go index 68dda4f..a7ea998 100644 --- a/api/bitbucket.go +++ b/api/bitbucket.go @@ -5,7 +5,6 @@ import ( "compress/gzip" "context" "encoding/json" - "errors" "io" "io/ioutil" "net/http" @@ -118,11 +117,6 @@ func (bb *BitBucketGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := bb.authenticate(w, r); err != nil { - handleError(unauthorizedError(err.Error()), w, r) - return - } - endpoint := config.BitBucket.Endpoint apiURL := singleJoiningSlash(endpoint, "/repositories/"+config.BitBucket.Repo) target, err := url.Parse(apiURL) @@ -142,39 +136,6 @@ func (bb *BitBucketGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { bb.proxy.ServeHTTP(w, r.WithContext(ctx)) } -func (bb *BitBucketGateway) authenticate(w http.ResponseWriter, r *http.Request) error { - ctx := r.Context() - claims := getClaims(ctx) - config := getConfig(ctx) - - if claims == nil { - return errors.New("Access to endpoint not allowed: no claims found in Bearer token") - } - - if !bitbucketAllowedRegexp.MatchString(r.URL.Path) { - return errors.New("Access to endpoint not allowed: this part of BitBucket's API has been restricted") - } - - if len(config.Roles) == 0 { - return nil - } - - roles, ok := claims.AppMetaData["roles"] - if ok { - roleStrings, _ := roles.([]interface{}) - for _, data := range roleStrings { - role, _ := data.(string) - for _, adminRole := range config.Roles { - if role == adminRole { - return nil - } - } - } - } - - return errors.New("Access to endpoint not allowed: your role doesn't allow access") -} - func rewriteBitBucketLink(link, endpointAPIURL, proxyAPIURL string) string { return proxyAPIURL + strings.TrimPrefix(link, endpointAPIURL) } diff --git a/api/github.go b/api/github.go index 8514017..f7bd599 100644 --- a/api/github.go +++ b/api/github.go @@ -1,7 +1,6 @@ package api import ( - "errors" "net/http" "net/http/httputil" "net/url" @@ -60,11 +59,6 @@ func (gh *GitHubGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := gh.authenticate(w, r); err != nil { - handleError(unauthorizedError(err.Error()), w, r) - return - } - endpoint := config.GitHub.Endpoint apiURL := singleJoiningSlash(endpoint, "/repos/"+config.GitHub.Repo) target, err := url.Parse(apiURL) @@ -80,43 +74,6 @@ func (gh *GitHubGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { gh.proxy.ServeHTTP(w, r.WithContext(ctx)) } -func (gh *GitHubGateway) authenticate(w http.ResponseWriter, r *http.Request) error { - ctx := r.Context() - claims := getClaims(ctx) - config := getConfig(ctx) - - log := getLogEntry(r) - log.Infof("authenticate context: %v+", ctx) - if claims == nil { - // @TODO? WARNING: the check should be done in auth.go, imo. - // Having the jwt in the context (and thus, sent to github.com) is not necessary - // return errors.New("Access to endpoint not allowed: no claims found in Bearer token") - } - - if !allowedRegexp.MatchString(r.URL.Path) { - return errors.New("Access to endpoint not allowed: this part of GitHub's API has been restricted") - } - - if len(config.Roles) == 0 { - return nil - } - - roles, ok := claims.AppMetaData["roles"] - if ok { - roleStrings, _ := roles.([]interface{}) - for _, data := range roleStrings { - role, _ := data.(string) - for _, adminRole := range config.Roles { - if role == adminRole { - return nil - } - } - } - } - - return errors.New("Access to endpoint not allowed: your role doesn't allow access") -} - type GitHubTransport struct{} func (t *GitHubTransport) RoundTrip(r *http.Request) (*http.Response, error) { diff --git a/api/gitlab.go b/api/gitlab.go index 3702eae..4cad06b 100644 --- a/api/gitlab.go +++ b/api/gitlab.go @@ -1,7 +1,6 @@ package api import ( - "errors" "net/http" "net/http/httputil" "net/url" @@ -79,11 +78,6 @@ func (gl *GitLabGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := gl.authenticate(w, r); err != nil { - handleError(unauthorizedError(err.Error()), w, r) - return - } - endpoint := config.GitLab.Endpoint // repos in the form of userName/repoName must be encoded as // userName%2FrepoName @@ -99,39 +93,6 @@ func (gl *GitLabGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { gl.proxy.ServeHTTP(w, r.WithContext(ctx)) } -func (gl *GitLabGateway) authenticate(w http.ResponseWriter, r *http.Request) error { - ctx := r.Context() - claims := getClaims(ctx) - config := getConfig(ctx) - - if claims == nil { - return errors.New("Access to endpoint not allowed: no claims found in Bearer token") - } - - if !gitlabAllowedRegexp.MatchString(r.URL.Path) { - return errors.New("Access to endpoint not allowed: this part of GitLab's API has been restricted") - } - - if len(config.Roles) == 0 { - return nil - } - - roles, ok := claims.AppMetaData["roles"] - if ok { - roleStrings, _ := roles.([]interface{}) - for _, data := range roleStrings { - role, _ := data.(string) - for _, adminRole := range config.Roles { - if role == adminRole { - return nil - } - } - } - } - - return errors.New("Access to endpoint not allowed: your role doesn't allow access") -} - var gitlabLinkRegex = regexp.MustCompile("<(.*?)>") var gitlabLinkRelRegex = regexp.MustCompile("rel=\"(.*?)\"") diff --git a/api/middleware.go b/api/middleware.go index adf9ca5..08cae70 100644 --- a/api/middleware.go +++ b/api/middleware.go @@ -80,7 +80,7 @@ func (a *API) verifyOperatorRequest(w http.ResponseWriter, req *http.Request) (c } func (a *API) extractOperatorRequest(w http.ResponseWriter, req *http.Request) (context.Context, string, error) { - token, err := a.extractBearerToken(w, req) + token, err := a.auth.extractBearerToken(w, req) if err != nil { return nil, token, err } From 5f0f6a4f096960cb52e655b969d6731f7c4f55df Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Tue, 27 Nov 2018 13:24:57 -0800 Subject: [PATCH 10/15] ORGANIC-467. Restored authorize() capability. --- api/api.go | 8 ++++---- api/auth.go | 23 ++++++++++++++--------- api/bitbucket.go | 5 +++++ api/github.go | 5 +++++ api/gitlab.go | 5 +++++ 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/api/api.go b/api/api.go index c980997..17a0258 100644 --- a/api/api.go +++ b/api/api.go @@ -77,10 +77,10 @@ func NewAPIWithVersion(ctx context.Context, globalConfig *conf.GlobalConfigurati r.Use(api.loadJWSSignatureHeader) r.Use(api.loadInstanceConfig) } - r.With(api.auth.authenticate).Mount("/github", NewGitHubGateway()) - r.With(api.auth.authenticate).Mount("/gitlab", NewGitLabGateway()) - r.With(api.auth.authenticate).Mount("/bitbucket", NewBitBucketGateway()) - r.With(api.auth.authenticate).Get("/settings", api.Settings) + r.With(api.auth.accessControl).Mount("/github", NewGitHubGateway()) + r.With(api.auth.accessControl).Mount("/gitlab", NewGitLabGateway()) + r.With(api.auth.accessControl).Mount("/bitbucket", NewBitBucketGateway()) + r.With(api.auth.accessControl).Get("/settings", api.Settings) }) if globalConfig.MultiInstanceMode { diff --git a/api/auth.go b/api/auth.go index bf00f30..9d163aa 100644 --- a/api/auth.go +++ b/api/auth.go @@ -2,7 +2,6 @@ package api import ( "context" - "errors" "net/http" "github.com/netlify/git-gateway/conf" @@ -15,7 +14,17 @@ type Auth struct { version string } -// authenicate checks incoming requests for tokens presented using the Authorization header +// check both authentication and authorization +func (a *Auth) accessControl(w http.ResponseWriter, r *http.Request) (context.Context, error) { + _, err := a.authenticate(w, r) + if err != nil { + return nil, err + } + + return a.authorize(w, r) +} + +// authenticate checks incoming requests for tokens presented using the Authorization header func (a *Auth) authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) { logrus.Info("Getting auth token") token, err := a.extractBearerToken(w, r) @@ -33,13 +42,9 @@ func (a *Auth) authorize(w http.ResponseWriter, r *http.Request) (context.Contex claims := getClaims(ctx) config := getConfig(ctx) - logrus.Infof("authenticate context: %v+", ctx) + logrus.Infof("authenticate url: %v+", r.URL) if claims == nil { - return nil, errors.New("Access to endpoint not allowed: no claims found in Bearer token") - } - - if !allowedRegexp.MatchString(r.URL.Path) { - return nil, errors.New("Access to endpoint not allowed: this part of GitHub's API has been restricted") + return nil, unauthorizedError("Access to endpoint not allowed: no claims found in Bearer token") } if len(config.Roles) == 0 { @@ -59,7 +64,7 @@ func (a *Auth) authorize(w http.ResponseWriter, r *http.Request) (context.Contex } } - return nil, errors.New("Access to endpoint not allowed: your role doesn't allow access") + return nil, unauthorizedError("Access to endpoint not allowed: your role doesn't allow access") } func NewAuthWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, version string) *Auth { diff --git a/api/bitbucket.go b/api/bitbucket.go index a7ea998..1f30dc4 100644 --- a/api/bitbucket.go +++ b/api/bitbucket.go @@ -117,6 +117,11 @@ func (bb *BitBucketGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if !bitbucketAllowedRegexp.MatchString(r.URL.Path) { + handleError(unauthorizedError("Access to endpoint not allowed: this part of BitBucket's API has been restricted"), w, r) + return + } + endpoint := config.BitBucket.Endpoint apiURL := singleJoiningSlash(endpoint, "/repositories/"+config.BitBucket.Repo) target, err := url.Parse(apiURL) diff --git a/api/github.go b/api/github.go index f7bd599..a8925b0 100644 --- a/api/github.go +++ b/api/github.go @@ -59,6 +59,11 @@ func (gh *GitHubGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if !allowedRegexp.MatchString(r.URL.Path) { + handleError(unauthorizedError("Access to endpoint not allowed: this part of GitHub's API has been restricted"), w, r) + return + } + endpoint := config.GitHub.Endpoint apiURL := singleJoiningSlash(endpoint, "/repos/"+config.GitHub.Repo) target, err := url.Parse(apiURL) diff --git a/api/gitlab.go b/api/gitlab.go index 4cad06b..7c855a7 100644 --- a/api/gitlab.go +++ b/api/gitlab.go @@ -78,6 +78,11 @@ func (gl *GitLabGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if !gitlabAllowedRegexp.MatchString(r.URL.Path) { + handleError(unauthorizedError("Access to endpoint not allowed: this part of GitLab's API has been restricted"), w, r) + return + } + endpoint := config.GitLab.Endpoint // repos in the form of userName/repoName must be encoded as // userName%2FrepoName From 163a7085d4188463b91421d17b219955999d60dc Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Tue, 27 Nov 2018 19:04:58 -0800 Subject: [PATCH 11/15] ORGANIC-467. Added Okta JWT parser "properly", mostly. --- api/auth.go | 177 +++++++++++++++++++++++++++++++++---------------- api/context.go | 27 +++----- api/github.go | 2 - 3 files changed, 128 insertions(+), 78 deletions(-) diff --git a/api/auth.go b/api/auth.go index 9d163aa..c4d2535 100644 --- a/api/auth.go +++ b/api/auth.go @@ -4,30 +4,88 @@ import ( "context" "net/http" + jwt "github.com/dgrijalva/jwt-go" "github.com/netlify/git-gateway/conf" "github.com/sirupsen/logrus" "github.com/okta/okta-jwt-verifier-golang" ) +type Authenticator interface { + // authenticate checks incoming requests for tokens presented using the Authorization header + authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) + getName() string +} + +type Authorizer interface { + // authorize checks incoming requests for roles data in tokens that is parsed and verified by prior authentication step + authorize(w http.ResponseWriter, r *http.Request) (context.Context, error) + getName() string +} + type Auth struct { config *conf.GlobalConfiguration + authenticator Authenticator + authorizer Authorizer version string } +type JWTAuthenticator struct { + name string + auth Auth +} + +type OktaJWTAuthenticator struct { + name string + auth Auth +} + +type RolesAuthorizer struct { + name string + auth Auth +} + +func NewAuthWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, version string) *Auth { + auth := &Auth{config: globalConfig, version: version} + + auth.authenticator = &OktaJWTAuthenticator{name: "bearer-jwt-token", auth: *auth} + auth.authorizer = &RolesAuthorizer{name: "bearer-jwt-token-roles", auth: *auth} + + return auth +} + // check both authentication and authorization func (a *Auth) accessControl(w http.ResponseWriter, r *http.Request) (context.Context, error) { - _, err := a.authenticate(w, r) + logrus.Infof("Authenticate with: %v", a.authenticator.getName()) + ctx, err := a.authenticator.authenticate(w, r) if err != nil { return nil, err } - return a.authorize(w, r) + logrus.Infof("Authorizing with: %v", a.authorizer.getName()) + return a.authorizer.authorize(w, r.WithContext(ctx)) +} + +func (a *Auth) extractBearerToken(w http.ResponseWriter, r *http.Request) (string, error) { + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + return "", unauthorizedError("This endpoint requires a Bearer token") + } + + matches := bearerRegexp.FindStringSubmatch(authHeader) + if len(matches) != 2 { + return "", unauthorizedError("This endpoint requires a Bearer token") + } + + return matches[1], nil +} + +func (a *JWTAuthenticator) getName() string { + return a.name } -// authenticate checks incoming requests for tokens presented using the Authorization header -func (a *Auth) authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) { +func (a *JWTAuthenticator) authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) { logrus.Info("Getting auth token") - token, err := a.extractBearerToken(w, r) + token, err := a.auth.extractBearerToken(w, r) if err != nil { return nil, err } @@ -36,61 +94,36 @@ func (a *Auth) authenticate(w http.ResponseWriter, r *http.Request) (context.Con return a.parseJWTClaims(token, r) } -// authorize checks incoming requests for roles data in tokens that is parsed and verified by prior authentication step -func (a *Auth) authorize(w http.ResponseWriter, r *http.Request) (context.Context, error) { - ctx := r.Context() - claims := getClaims(ctx) - config := getConfig(ctx) - - logrus.Infof("authenticate url: %v+", r.URL) - if claims == nil { - return nil, unauthorizedError("Access to endpoint not allowed: no claims found in Bearer token") - } - - if len(config.Roles) == 0 { - return ctx, nil - } +func (a *JWTAuthenticator) parseJWTClaims(bearer string, r *http.Request) (context.Context, error) { + config := getConfig(r.Context()) + p := jwt.Parser{ValidMethods: []string{jwt.SigningMethodHS256.Name}} + token, err := p.ParseWithClaims(bearer, &GatewayClaims{}, func(token *jwt.Token) (interface{}, error) { + return []byte(config.JWT.Secret), nil + }) - roles, ok := claims.AppMetaData["roles"] - if ok { - roleStrings, _ := roles.([]interface{}) - for _, data := range roleStrings { - role, _ := data.(string) - for _, adminRole := range config.Roles { - if role == adminRole { - return ctx, nil - } - } - } + if err != nil { + return nil, unauthorizedError("Invalid token: %v", err) } - - return nil, unauthorizedError("Access to endpoint not allowed: your role doesn't allow access") + claims := token.Claims.(GatewayClaims) + return withClaims(r.Context(), &claims), nil } -func NewAuthWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, version string) *Auth { - auth := &Auth{config: globalConfig, version: version} - - return auth +func (a *OktaJWTAuthenticator) getName() string { + return a.name } -func (a *Auth) extractBearerToken(w http.ResponseWriter, r *http.Request) (string, error) { - authHeader := r.Header.Get("Authorization") - if authHeader == "" { - return "", unauthorizedError("This endpoint requires a Bearer token") - } - - matches := bearerRegexp.FindStringSubmatch(authHeader) - if len(matches) != 2 { - return "", unauthorizedError("This endpoint requires a Bearer token") +func (a *OktaJWTAuthenticator) authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) { + logrus.Info("Getting auth token") + token, err := a.auth.extractBearerToken(w, r) + if err != nil { + return nil, err } - return matches[1], nil + logrus.Infof("Parsing JWT claims: %v", token) + return a.parseOktaJWTClaims(token, r) } -func (a *Auth) parseJWTClaims(bearer string, r *http.Request) (context.Context, error) { - // Reimplemented to use Okta lib - // Original validation only work for HS256 algo, - // Okta supports RS256 only which requires public key downloading and caching (key rotation) +func (a *OktaJWTAuthenticator) parseOktaJWTClaims(bearer string, r *http.Request) (context.Context, error) { config := getConfig(r.Context()) toValidate := map[string]string{} @@ -106,17 +139,47 @@ func (a *Auth) parseJWTClaims(bearer string, r *http.Request) (context.Context, _, err := verifier.VerifyAccessToken(bearer) - // @TODO? WARNING: Should be roles and other claims be checked here? - if err != nil { return nil, unauthorizedError("Invalid token: %v", err) } + claims := GatewayClaims{Email: "e", StandardClaims: jwt.StandardClaims{Audience: "a"}} + logrus.Infof("parseJWTClaims passed") + return withClaims(r.Context(), &claims), nil +} - // return nil, because the `github.go` is coded to send personal token - // both github oauth generates its own id, so oauth pass-thru is impossible - // we can improve the gateway to talk oauth with github.com, but we will - // still return nil here. - return nil, nil +func (a *RolesAuthorizer) getName() string { + return a.name +} + +func (a *RolesAuthorizer) authorize(w http.ResponseWriter, r *http.Request) (context.Context, error) { + ctx := r.Context() + claims := getClaims(ctx) + config := getConfig(ctx) + + logrus.Infof("authenticate url: %v+", r.URL) + logrus.Infof("claims: %v+", claims) + if claims == nil { + return nil, unauthorizedError("Access to endpoint not allowed: no claims found in Bearer token") + } + + if len(config.Roles) == 0 { + return ctx, nil + } + + roles, ok := claims.AppMetaData["roles"] + if ok { + roleStrings, _ := roles.([]interface{}) + for _, data := range roleStrings { + role, _ := data.(string) + for _, adminRole := range config.Roles { + if role == adminRole { + return ctx, nil + } + } + } + } + + return nil, unauthorizedError("Access to endpoint not allowed: your role doesn't allow access") } diff --git a/api/context.go b/api/context.go index 583d825..ca0abed 100644 --- a/api/context.go +++ b/api/context.go @@ -4,7 +4,6 @@ import ( "context" "net/url" - jwt "github.com/dgrijalva/jwt-go" "github.com/netlify/git-gateway/conf" "github.com/netlify/git-gateway/models" ) @@ -20,8 +19,8 @@ func (c contextKey) String() string { } const ( - accessTokenKey = contextKey("token") - tokenKey = contextKey("jwt") + accessTokenKey = contextKey("access_token") + tokenClaimsKey = contextKey("jwt_claims") requestIDKey = contextKey("request_id") configKey = contextKey("config") instanceIDKey = contextKey("instance_id") @@ -31,27 +30,17 @@ const ( netlifyIDKey = contextKey("netlify_id") ) -// withToken adds the JWT token to the context. -func withToken(ctx context.Context, token *jwt.Token) context.Context { - return context.WithValue(ctx, tokenKey, token) -} - -// getToken reads the JWT token from the context. -func getToken(ctx context.Context) *jwt.Token { - obj := ctx.Value(tokenKey) - if obj == nil { - return nil - } - - return obj.(*jwt.Token) +// withTokenClaims adds the JWT token claims to the context. +func withClaims(ctx context.Context, claims *GatewayClaims) context.Context { + return context.WithValue(ctx, tokenClaimsKey, claims) } func getClaims(ctx context.Context) *GatewayClaims { - token := getToken(ctx) - if token == nil { + claims := ctx.Value(tokenClaimsKey) + if claims == nil { return nil } - return token.Claims.(*GatewayClaims) + return claims.(*GatewayClaims) } func withRequestID(ctx context.Context, id string) context.Context { diff --git a/api/github.go b/api/github.go index a8925b0..47cedb5 100644 --- a/api/github.go +++ b/api/github.go @@ -74,8 +74,6 @@ func (gh *GitHubGateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx = withProxyTarget(ctx, target) ctx = withAccessToken(ctx, config.GitHub.AccessToken) - log := getLogEntry(r) - log.Infof("proxy.ServeHTTP: %+v\n", r.WithContext(ctx)) gh.proxy.ServeHTTP(w, r.WithContext(ctx)) } From 76f77fe92a92a62093d014ef1c8b1cfa7f2d5bdc Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Tue, 27 Nov 2018 20:48:39 -0800 Subject: [PATCH 12/15] ORGANIC-467. Added the option to specify Authenticator to use in env. --- api/api.go | 2 +- api/auth.go | 25 ++++++++++++++++------- conf/configuration.go | 1 + example.env | 46 +++++++++++++++++++++++++------------------ 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/api/api.go b/api/api.go index 17a0258..e537ab4 100644 --- a/api/api.go +++ b/api/api.go @@ -59,7 +59,7 @@ func NewAPI(globalConfig *conf.GlobalConfiguration, db storage.Connection) *API // NewAPIWithVersion creates a new REST API using the specified version func NewAPIWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, db storage.Connection, version string) *API { - auth := NewAuthWithVersion(ctx, globalConfig, version) + auth := NewAuthWithVersion(ctx, version) api := &API{config: globalConfig, db: db, auth: *auth, version: version} xffmw, _ := xff.Default() diff --git a/api/auth.go b/api/auth.go index c4d2535..86bf547 100644 --- a/api/auth.go +++ b/api/auth.go @@ -5,25 +5,23 @@ import ( "net/http" jwt "github.com/dgrijalva/jwt-go" - "github.com/netlify/git-gateway/conf" "github.com/sirupsen/logrus" "github.com/okta/okta-jwt-verifier-golang" ) type Authenticator interface { - // authenticate checks incoming requests for tokens presented using the Authorization header + // `authenticate` checks incoming requests for tokens presented using the Authorization header authenticate(w http.ResponseWriter, r *http.Request) (context.Context, error) getName() string } type Authorizer interface { - // authorize checks incoming requests for roles data in tokens that is parsed and verified by prior authentication step + // `authorize` checks incoming requests for roles data in tokens that is parsed and verified by a prior `authenticate` step authorize(w http.ResponseWriter, r *http.Request) (context.Context, error) getName() string } type Auth struct { - config *conf.GlobalConfiguration authenticator Authenticator authorizer Authorizer version string @@ -44,10 +42,23 @@ type RolesAuthorizer struct { auth Auth } -func NewAuthWithVersion(ctx context.Context, globalConfig *conf.GlobalConfiguration, version string) *Auth { - auth := &Auth{config: globalConfig, version: version} +func NewAuthWithVersion(ctx context.Context, version string) *Auth { + config := getConfig(ctx) + auth := &Auth{version: version} + authenticatorName := config.JWT.Authenticator + + if (authenticatorName == "bearer-jwt-token") { + auth.authenticator = &JWTAuthenticator{name: "bearer-jwt-token", auth: *auth} + } else if (authenticatorName == "bearer-okta-jwt-token") { + auth.authenticator = &OktaJWTAuthenticator{name: "bearer-okta-jwt-token", auth: *auth} + } else { + if (authenticatorName != "") { + logrus.Fatal("Authenticator `%v` is not recognized", authenticatorName) + } else { + logrus.Fatal("Authenticator is not defined") + } + } - auth.authenticator = &OktaJWTAuthenticator{name: "bearer-jwt-token", auth: *auth} auth.authorizer = &RolesAuthorizer{name: "bearer-jwt-token-roles", auth: *auth} return auth diff --git a/conf/configuration.go b/conf/configuration.go index 6c6b688..80495f2 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -49,6 +49,7 @@ type JWTConfiguration struct { CID string `envconfig:"CLIENT_ID" json:"client_id,omitempty"` Issuer string `envconfig:"ISSUER" json:"issuer,omitempty"` AUD string `envconfig:"AUD" json:"aud,omitempty"` + Authenticator string `envconfig:"AUTHENTICATOR" json:"authenticator,omitempty"` } // GlobalConfiguration holds all the configuration that applies to all instances. diff --git a/example.env b/example.env index 5a5ca04..864edda 100644 --- a/example.env +++ b/example.env @@ -1,30 +1,38 @@ -# Warning: Many configuration would not work with quote (ie ""). +# Do not use quote (ie, "") if you start `git-gateway` with docker command: `--env-file` -# JWT Secret is not needed for RS256. Instead, issuer should be specified -# (eg, https://dev-1234.oktapreview.com/oauth2/default) -GITGATEWAY_JWT_SECRET= +# DB +GITGATEWAY_DB_DRIVER=sqlite3 +DATABASE_URL=gorm.db -# @TODO - REQUIRED for Okta -ISSUER= +# Startup Options +GITGATEWAY_API_HOST=localhost +PORT=9999 -# @TODO - REQUIRED for Okta -CLIENT_ID= +# <> config for JWT Token with HS256 alg +# AUTHENTICATOR=bearer-jwt-token + +# Leave blank for other AUTHENTICATOR +GITGATEWAY_JWT_SECRET="CHANGE-THIS! VERY IMPORTANT!" +# +# <> config for JWT Token with Okta (RS256) alg +AUTHENTICATOR=bearer-okta-jwt-token + +# REQUIRED for AUTHENTICATOR=bearer-okta-jwt-token AUD=api://default -GITGATEWAY_DB_DRIVER=sqlite3 -DATABASE_URL=gorm.db +# REQUIRED for AUTHENTICATOR=bearer-okta-jwt-token +ISSUER= +# -# @TODO - Is there way to expose internal port from Docker? -GITGATEWAY_API_HOST=0.0.0.0 -PORT=8087 +# REQUIRED for both AUTHENTICATOR = {bearer-jwt-token or bearer-okta-jwt-token} +CLIENT_ID= -# @TODO - REQUIRED +# REQUIRED for GITHUB GITGATEWAY_GITHUB_ACCESS_TOKEN= -# @TODO - REQUIRED -GITGATEWAY_GITHUB_REPO= +# REQUIRED for GITHUB +GITGATEWAY_GITHUB_REPO=owner/name -# Original example.env wrote: leave blank to allow all roles. But, it won't -# work unless it is commented out -# GITGATEWAY_ROLES= +# Commented out to allow roles +GITGATEWAY_ROLES=admin,cms From ab11d60c897edfefe38b83065345a79ce1342d7c Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Wed, 28 Nov 2018 11:38:07 -0800 Subject: [PATCH 13/15] ORGANIC-467. Added back golint. --- Dockerfile | 4 ++-- Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f28c7f4..a7af9ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,9 @@ RUN apk add --update alpine-sdk RUN rm -rf /var/cache/apk/* -RUN go get github.com/Masterminds/glide +RUN go get -u github.com/Masterminds/glide golang.org/x/lint/golint -RUN adduser -D -u 1000 netlify && cd /go/src/github.com/netlify/git-gateway && make deps build && mv git-gateway /usr/local/bin/ +RUN adduser -D -u 1000 netlify && cd /go/src/github.com/netlify/git-gateway && make deps build lint && mv git-gateway /usr/local/bin/ USER netlify CMD ["git-gateway"] diff --git a/Makefile b/Makefile index 6109281..bdf5ffb 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CHECK_FILES?=$$(go list ./... | grep -v /vendor/) help: ## Show this help. @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) -all: vet test build ## Run the tests and build the binary. +all: lint vet test build ## Run the tests and build the binary. build: ## Build the binary. go build -ldflags "-X github.com/netlify/git-gateway/cmd.Version=`git rev-parse HEAD`" From 8e77f810586b0f21426279a24bc641b4525e92d7 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Wed, 28 Nov 2018 12:22:28 -0800 Subject: [PATCH 14/15] ORGANIC-467. Updated README.md to make the instruction more general for public use. --- README.md | 52 +++++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e7d263b..4e48fce 100644 --- a/README.md +++ b/README.md @@ -30,48 +30,38 @@ for GitLab: /repos/:owner/:name/tree/ ``` -**Running `git-gateway`** -**(Do not merge this section back to the open source project)** -**(Do not deploy it to production. It is a Proof of Concept has has not been secured. See @TODO items in code.** -**(the instruction assume Okta, and github.com)** +### Trying out `git-gateway` + +The instructions below is a way of testing out `git-gateway`. It assumes you have Docker installed and are familiar with Okta (an IDaaS). If you are using a different stack, please adjust the steps accordingly. 1. pull down this project 2. generate a `personal access token` on github. (recommended: using a test account and w/ `repo:status` and `public_repo` permission only) https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ 3. `cp example.env my.env` 4. update `GITGATEWAY_GITHUB_ACCESS_TOKEN` value in `my.env` accordingly -5. update `GITGATEWAY_GITHUB_REPO` value in `my.env` (it will be where the content being stored, eg, `owner/netlify-test`.) +5. update `GITGATEWAY_GITHUB_REPO` value in `my.env` (it will be where the content being stored, eg, `owner/netlify-cms-storage`.) 6. sign up for a Dev account on Okta: https://developer.okta.com/signup/ 7. create a SPA Application onto the Dev account: a. fill out the details - b. Pick "Send ID Token directly to app (Okta Simplified)" - c. have redirect uri points to the url of your content-cms ip:port - (eg, `http://localhost:8080/admin` etc, see, https://github.com/<< your org >>/content-cms) + b. pick `Either Okta or App` + c. pick `Send ID Token directly to app (Okta Simplified)`` + d. have redirect uri points to the url of your `my-netlify-cms` ip:port + (eg, `http://localhost:8080/admin` etc, see, https://github.com/<< your org >>/my-netlify-cms) + e. make sure `Authorization Servers` is activated + f. go to `Trusted Origins` tab and add the url for your `my-netlify-cms` instance + g. add yourself or a test user 8. update `ISSUER` value in `my.env` accordingly (eg, `https://dev-1234.oktapreview.com/oauth2/default`) 9. update `CLIENT_ID` value in `my.env` accordingly (eg, `32q897q234q324rq42322q`) -10. install Docker and add the `localdev` network -11. inspect Dockfile and then build the docker with this command: +10. comment out `GITGATEWAY_ROLES` to disable role checking (authorization is controlled by `Assignments` on Okta) +11. update `GITGATEWAY_API_HOST` to `0.0.0.0` +12. inspect Dockerfile and then build the docker with this command: `docker build -t netlify/git-gateway:latest .` -12. run `git-gateway` with this command: - `docker run --rm --env-file my.env --net localdev -p 127.0.0.1:8087:8087 --expose 8087 -ti --name netlify-git-gateway "netlify/git-gateway:latest"` -13. update `config.yml` in your content-cms repo (ie, https://github.com/<< your org >>/content-cms). +13. run `git-gateway` with this command: + `docker run --rm --env-file my.env -p 127.0.0.1:9999:9999 --expose 9999 -ti --name netlify-git-gateway "netlify/git-gateway:latest"` +14. update `config.yml` in your my-netlify-cms repo. change `backend.name` value to `git-gateway` - change `backend.gateway_url` value to `http://localhost:8087` -14. run `content-cms` following the README.md - -**Develop, Build and Run git-gateway** - -1. Follow instructions 1 - 10 in previous "Running `git-gateway`" section -2. Run these commands once: - ``` - docker build -t netlify/git-gateway:latest . - docker run --rm --env-file my.env --net localdev -p 127.0.0.1:8087:8087 --expose 8087 -ti -v $PWD:/go/src/github.com/netlify/git-gateway --entrypoint '/bin/sh' --user root netlify/git-gateway:latest - cd /go/src/github.com/netlify/git-gateway - make deps - ``` -3. Run these commands after edit: - ``` - make build && ./git-gateway - ``` -4. ` + c` to stop + change `backend.gateway_url` value to `http://localhost:9999` +15. integrate okta sign-in to your `my-netlify-cms` (eg, https://developer.okta.com/quickstart/#/widget/nodejs/express) +16. start your `my-netlify-cms` instance +See, Wiki page for additional information. From 9f585104b9d4645dd4b13c750423fbe93197bec2 Mon Sep 17 00:00:00 2001 From: Thomas Yip Date: Wed, 28 Nov 2018 12:56:10 -0800 Subject: [PATCH 15/15] ORGANIC-467. Populated Claims from Okta with real values. --- api/auth.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/api/auth.go b/api/auth.go index 86bf547..6a7f2c6 100644 --- a/api/auth.go +++ b/api/auth.go @@ -148,15 +148,28 @@ func (a *OktaJWTAuthenticator) parseOktaJWTClaims(bearer string, r *http.Request verifier := jwtVerifierSetup.New() - _, err := verifier.VerifyAccessToken(bearer) + token, err := verifier.VerifyAccessToken(bearer) if err != nil { return nil, unauthorizedError("Invalid token: %v", err) } + logrus.Infof("parseJWTClaims passed") - claims := GatewayClaims{Email: "e", StandardClaims: jwt.StandardClaims{Audience: "a"}} + claims := GatewayClaims{ + Email: token.Claims["sub"].(string), + AppMetaData: nil, + UserMetaData: nil, + StandardClaims: jwt.StandardClaims{ + Audience: token.Claims["aud"].(string), + ExpiresAt: int64(token.Claims["exp"].(float64)), + Id: token.Claims["jti"].(string), + IssuedAt: int64(token.Claims["iat"].(float64)), + Issuer: token.Claims["iss"].(string), + NotBefore: 0, + Subject: token.Claims["sub"].(string), + }, + } - logrus.Infof("parseJWTClaims passed") return withClaims(r.Context(), &claims), nil }