diff --git a/.ci-operator.yaml b/.ci-operator.yaml new file mode 100644 index 000000000000..e307e5af6628 --- /dev/null +++ b/.ci-operator.yaml @@ -0,0 +1,4 @@ +build_root_image: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.24-openshift-4.21 diff --git a/.github/workflows/asset-transparency.yaml b/.github/workflows/asset-transparency.yaml deleted file mode 100644 index 6f4f2bf0cfd9..000000000000 --- a/.github/workflows/asset-transparency.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Publish Release Assets to Asset Transparency Log - -on: - release: - types: [published, created, edited, released] - -jobs: - github_release_asset_transparency_log_publish_job: - runs-on: ubuntu-latest - name: Publish GitHub release asset digests to https://beta-asset.transparencylog.net - steps: - - name: Gather URLs from GitHub release and publish - id: asset-transparency - uses: transparencylog/github-releases-asset-transparency-verify-action@v11 - - name: List verified and published URLs - run: echo "Verified URLs ${{ steps.asset-transparency.outputs.verified }}" - - name: List failed URLs - run: echo "Failed URLs ${{ steps.asset-transparency.outputs.failed }}" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4ec326b52eda..ef9efac08359 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -11,22 +11,25 @@ jobs: - linux-386-e2e steps: - uses: actions/checkout@v2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - uses: actions/setup-go@v2 with: - go-version: "^1.16" + go-version: ${{ steps.goversion.outputs.goversion }} - run: date - env: TARGET: ${{ matrix.target }} run: | + set -euo pipefail + echo "${TARGET}" case "${TARGET}" in linux-amd64-e2e) - PASSES='build release e2e' MANUAL_VER=v3.4.7 CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log - ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log + make install-gofail + CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' FAILPOINTS='true' make test-e2e-release ;; linux-386-e2e) - GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log - ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log + GOARCH=386 CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' make test-e2e ;; *) echo "Failed to find target" diff --git a/.github/workflows/functional.yaml b/.github/workflows/functional.yaml index 0e3a46fc6881..176f90aa78bb 100644 --- a/.github/workflows/functional.yaml +++ b/.github/workflows/functional.yaml @@ -10,13 +10,17 @@ jobs: - linux-amd64-functional steps: - uses: actions/checkout@v2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - uses: actions/setup-go@v2 with: - go-version: "^1.16" + go-version: ${{ steps.goversion.outputs.goversion }} - run: date - env: TARGET: ${{ matrix.target }} run: | + set -euo pipefail + echo "${TARGET}" case "${TARGET}" in linux-amd64-functional) diff --git a/.github/workflows/gh-workflow-approve.yaml b/.github/workflows/gh-workflow-approve.yaml new file mode 100644 index 000000000000..c85d859e6b85 --- /dev/null +++ b/.github/workflows/gh-workflow-approve.yaml @@ -0,0 +1,41 @@ +--- +name: Approve GitHub Workflows +permissions: read-all + +on: + pull_request_target: + types: + - labeled + - synchronize + +jobs: + approve: + name: Approve ok-to-test + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Update PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} + script: | + const result = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + event: "pull_request", + status: "action_required", + head_sha: context.payload.pull_request.head.sha, + per_page: 100 + }); + + for (var run of result.data.workflow_runs) { + await github.rest.actions.approveWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id + }); + } diff --git a/.github/workflows/govuln.yaml b/.github/workflows/govuln.yaml new file mode 100644 index 000000000000..9ec94d1bab7e --- /dev/null +++ b/.github/workflows/govuln.yaml @@ -0,0 +1,21 @@ +--- +name: Go Vulnerability Checker +on: [push, pull_request] +permissions: read-all +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: date + - run: | + set -euo pipefail + + go install golang.org/x/vuln/cmd/govulncheck@latest + + find . -name go.mod | xargs -I'{}' /bin/bash -c 'echo scanning $(dirname {}); govulncheck -C $(dirname {}) -show verbose ./...' diff --git a/.github/workflows/grpcproxy.yaml b/.github/workflows/grpcproxy.yaml index 081d8be328f8..c4837eb41fc5 100644 --- a/.github/workflows/grpcproxy.yaml +++ b/.github/workflows/grpcproxy.yaml @@ -10,18 +10,21 @@ jobs: - linux-amd64-grpcproxy steps: - uses: actions/checkout@v2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - uses: actions/setup-go@v2 with: - go-version: "^1.16" + go-version: ${{ steps.goversion.outputs.goversion }} - run: date - env: TARGET: ${{ matrix.target }} run: | + set -euo pipefail + echo "${TARGET}" case "${TARGET}" in linux-amd64-grpcproxy) - PASSES='build grpcproxy' CPU='4' COVER='false' RACE='true' ./test.sh 2>&1 | tee test.log - ! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log + PASSES='build grpcproxy' CPU='4' COVER='false' RACE='true' ./test.sh ;; *) echo "Failed to find target" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000000..8fb9ab62926f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,75 @@ +name: Release +on: [push, pull_request] +jobs: + main: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: release + run: | + set -euo pipefail + + git config --global user.email "github-action@etcd.io" + git config --global user.name "Github Action" + gpg --batch --gen-key <> "$GITHUB_OUTPUT" + - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: | + set -euo pipefail + + make verify + - run: | + set -euo pipefail + + make fix + + DIFF=$(git status --porcelain) + + if [ -n "$DIFF" ]; then + echo "These files were modified:" + echo + echo "$DIFF" + echo + exit 1 + fi diff --git a/.github/workflows/tests-template.yaml b/.github/workflows/tests-template.yaml new file mode 100644 index 000000000000..2b5af12c4b8f --- /dev/null +++ b/.github/workflows/tests-template.yaml @@ -0,0 +1,76 @@ +--- +name: Reusable Tests Workflow +on: + workflow_call: + inputs: + arch: + required: true + type: string + runs-on: + required: true + type: string + targets: + required: false + type: string + +jobs: + test: + runs-on: ${{ inputs.runs-on }} + # this is to prevent arm64 jobs from running at forked projects + if: inputs.arch == 'amd64' || github.repository == 'etcd-io/etcd' + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(inputs.targets) }} + steps: + - uses: actions/checkout@v2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@v2 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: date + - env: + TARGET: ${{ matrix.target }} + run: | + set -euo pipefail + + echo "${TARGET}" + case "${TARGET}" in + linux-test-smoke) + GOARCH=${{ inputs.arch }} CPU=4 RACE='false' make test-smoke + ;; + linux-integration-1-cpu) + make install-gofail + GOARCH=${{ inputs.arch }} CPU=1 RACE='false' FAILPOINTS='true' make test-integration + ;; + linux-integration-2-cpu) + make install-gofail + GOARCH=${{ inputs.arch }} CPU=2 RACE='false' FAILPOINTS='true' make test-integration + ;; + linux-integration-4-cpu) + make install-gofail + GOARCH=${{ inputs.arch }} CPU=4 RACE='false' FAILPOINTS='true' make test-integration + ;; + linux-unit-4-cpu-race) + GOARCH=${{ inputs.arch }} RACE='true' CPU='4' GO_TEST_FLAGS='-p=2' make test-unit + ;; + linux-386-unit-1-cpu) + GOOS=linux GOARCH=386 CPU=1 GO_TEST_FLAGS='-p=4' make test-unit + ;; + all-build) + GOARCH=amd64 PASSES='build' ./test.sh + GOARCH=386 PASSES='build' ./test.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOOS=darwin GOARCH=amd64 ./build.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOOS=darwin GOARCH=arm64 ./build.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOOS=windows GOARCH=amd64 ./build.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=arm ./build.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=arm64 ./build.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=ppc64le ./build.sh + GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=s390x ./build.sh + ;; + *) + echo "Failed to find target" + exit 1 + ;; + esac diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cf63ec25a82d..6f9f63985921 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,60 +1,15 @@ name: Tests on: [push, pull_request] jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - target: - - linux-amd64-fmt - - linux-amd64-integration-1-cpu - - linux-amd64-integration-2-cpu - - linux-amd64-integration-4-cpu - - linux-amd64-unit-4-cpu-race - - all-build - - linux-386-unit-1-cpu - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: "^1.16" - - run: date - - env: - TARGET: ${{ matrix.target }} - run: | - echo "${TARGET}" - case "${TARGET}" in - linux-amd64-fmt) - GOARCH=amd64 PASSES='fmt bom dep' ./test.sh - ;; - linux-amd64-integration-1-cpu) - GOARCH=amd64 CPU=1 PASSES='integration' RACE='false' ./test.sh - ;; - linux-amd64-integration-2-cpu) - GOARCH=amd64 CPU=2 PASSES='integration' RACE='false' ./test.sh - ;; - linux-amd64-integration-4-cpu) - GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test.sh - ;; - linux-amd64-unit-4-cpu-race) - GOARCH=amd64 PASSES='unit' RACE='true' CPU='4' ./test.sh -p=2 - ;; - all-build) - GOARCH=amd64 PASSES='build' ./test.sh - GOARCH=386 PASSES='build' ./test.sh - GO_BUILD_FLAGS='-v -mod=readonly' GOOS=darwin GOARCH=amd64 ./build.sh - GO_BUILD_FLAGS='-v -mod=readonly' GOOS=windows GOARCH=amd64 ./build.sh - GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=arm ./build.sh - GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=arm64 ./build.sh - GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=ppc64le ./build.sh - GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=s390x ./build.sh - ;; - linux-386-unit-1-cpu) - GOARCH=386 PASSES='unit' RACE='false' CPU='1' ./test -p=4 - ;; - *) - echo "Failed to find target" - exit 1 - ;; - esac + amd64: + uses: ./.github/workflows/tests-template.yaml + with: + arch: amd64 + runs-on: ubuntu-latest + targets: "['linux-test-smoke', + 'linux-integration-1-cpu', + 'linux-integration-2-cpu', + 'linux-integration-4-cpu', + 'linux-unit-4-cpu-race', + 'linux-386-unit-1-cpu', + 'all-build']" diff --git a/.github/workflows/verify-released-assets.yaml b/.github/workflows/verify-released-assets.yaml new file mode 100644 index 000000000000..3db5e98c8f83 --- /dev/null +++ b/.github/workflows/verify-released-assets.yaml @@ -0,0 +1,44 @@ +--- +name: Verify released binary assets +permissions: read-all + +on: + release: + types: [published] + +jobs: + verify-assets: + name: Verify released binary assets + runs-on: ubuntu-latest + steps: + - name: Verify binary assets + env: + GH_TOKEN: ${{ github.token }} + RELEASE: ${{ github.event.release.tag_name }} + REPOSITORY: ${{ github.repository }} + run: | + mkdir github-assets + pushd github-assets + gh --repo "${REPOSITORY}" release download "${RELEASE}" + + test_assets() { + if [ "$(wc -l - case "${TARGET}" in - linux-amd64-coverage) - sudo HOST_TMP_DIR=/tmp TEST_OPTS="VERBOSE='1'" make docker-test-coverage - ;; - linux-amd64-fmt-unit-go-tip-2-cpu) - GOARCH=amd64 PASSES='fmt unit' CPU='2' RACE='false' ./test.sh -p=2 - ;; - esac diff --git a/ADOPTERS.md b/ADOPTERS.md deleted file mode 100644 index c6c294637d32..000000000000 --- a/ADOPTERS.md +++ /dev/null @@ -1,250 +0,0 @@ ---- -title: Production users ---- - -This document tracks people and use cases for etcd in production. By creating a list of production use cases we hope to build a community of advisors that we can reach out to with experience using various etcd applications, operation environments, and cluster sizes. The etcd development team may reach out periodically to check-in on how etcd is working in the field and update this list. - -## All Kubernetes Users - -- *Application*: https://kubernetes.io/ -- *Environments*: AWS, OpenStack, Azure, Google Cloud, Huawei Cloud, Bare Metal, etc - -**This is a meta user; please feel free to document specific Kubernetes clusters!** - -All Kubernetes clusters use etcd as their primary data store. This means etcd's users include such companies as [Niantic, Inc Pokemon Go](https://cloudplatform.googleblog.com/2016/09/bringing-Pokemon-GO-to-life-on-Google-Cloud.html), [Box](https://blog.box.com/blog/kubernetes-box-microservices-maximum-velocity/), [CoreOS](https://coreos.com/tectonic), [Ticketmaster](https://www.youtube.com/watch?v=wqXVKneP0Hg), [Salesforce](https://www.salesforce.com) and many many more. - -## discovery.etcd.io - -- *Application*: https://github.com/coreos/discovery.etcd.io -- *Launched*: Feb. 2014 -- *Cluster Size*: 5 members, 5 discovery proxies -- *Order of Data Size*: 100s of Megabytes -- *Operator*: CoreOS, brandon.philips@coreos.com -- *Environment*: AWS -- *Backups*: Periodic async to S3 - -discovery.etcd.io is the longest continuously running etcd backed service that we know about. It is the basis of automatic cluster bootstrap and was launched in Feb. 2014: https://coreos.com/blog/etcd-0.3.0-released/. - -## OpenTable - -- *Application*: OpenTable internal service discovery and cluster configuration management -- *Launched*: May 2014 -- *Cluster Size*: 3 members each in 6 independent clusters; approximately 50 nodes reading / writing -- *Order of Data Size*: 10s of MB -- *Operator*: OpenTable, Inc; sschlansker@opentable.com -- *Environment*: AWS, VMWare -- *Backups*: None, all data can be re-created if necessary. - -## cycoresys.com - -- *Application*: multiple -- *Launched*: Jul. 2014 -- *Cluster Size*: 3 members, _n_ proxies -- *Order of Data Size*: 100s of kilobytes -- *Operator*: CyCore Systems, Inc, sys@cycoresys.com -- *Environment*: Baremetal -- *Backups*: Periodic sync to Ceph RadosGW and DigitalOcean VM - -CyCore Systems provides architecture and engineering for computing systems. This cluster provides microservices, virtual machines, databases, storage clusters to a number of clients. It is built on CoreOS machines, with each machine in the cluster running etcd as a peer or proxy. - -## Radius Intelligence - -- *Application*: multiple internal tools, Kubernetes clusters, bootstrappable system configs -- *Launched*: June 2015 -- *Cluster Size*: 2 clusters of 5 and 3 members; approximately a dozen nodes read/write -- *Order of Data Size*: 100s of kilobytes -- *Operator*: Radius Intelligence; jcderr@radius.com -- *Environment*: AWS, CoreOS, Kubernetes -- *Backups*: None, all data can be recreated if necessary. - -Radius Intelligence uses Kubernetes running CoreOS to containerize and scale internal toolsets. Examples include running [JetBrains TeamCity][teamcity] and internal AWS security and cost reporting tools. etcd clusters back these clusters as well as provide some basic environment bootstrapping configuration keys. - -## Vonage - -- *Application*: kubernetes, vault backend, system configuration for microservices, scheduling, locks (future - service discovery) -- *Launched*: August 2015 -- *Cluster Size*: 2 clusters of 5 members in 2 DCs, n local proxies 1-to-1 with microservice, (ssl and SRV look up) -- *Order of Data Size*: kilobytes -- *Operator*: Vonage [devAdmin][raoofm] -- *Environment*: VMWare, AWS -- *Backups*: Daily snapshots on VMs. Backups done for upgrades. - -## PD - -- *Application*: embed etcd -- *Launched*: Mar 2016 -- *Cluster Size*: 3 or 5 members -- *Order of Data Size*: megabytes -- *Operator*: PingCAP, Inc. -- *Environment*: Bare Metal, AWS, etc. -- *Backups*: None. - -PD(Placement Driver) is the central controller in the TiDB cluster. It saves the cluster meta information, schedule the data, allocate the global unique timestamp for the distributed transaction, etc. It embeds etcd to supply high availability and auto failover. - -## Huawei - -- *Application*: System configuration for overlay network (Canal) -- *Launched*: June 2016 -- *Cluster Size*: 3 members for each cluster -- *Order of Data Size*: kilobytes -- *Operator*: Huawei Euler Department -- *Environment*: [Huawei Cloud](http://www.hwclouds.com/product/cce.html) -- *Backups*: None, all data can be recreated if necessary. - -[teamcity]: https://www.jetbrains.com/teamcity/ -[raoofm]:https://github.com/raoofm - -## Qiniu Cloud - -- *Application*: system configuration for microservices, distributed locks -- *Launched*: Jan. 2016 -- *Cluster Size*: 3 members each with several clusters -- *Order of Data Size*: kilobytes -- *Operator*: Pandora, chenchao@qiniu.com -- *Environment*: Baremetal -- *Backups*: None, all data can be recreated if necessary - -## QingCloud - -- *Application*: [QingCloud][qingcloud] appcenter cluster for service discovery as [metad][metad] backend. -- *Launched*: December 2016 -- *Cluster Size*: 1 cluster of 3 members per user. -- *Order of Data Size*: kilobytes -- *Operator*: [yunify][yunify] -- *Environment*: QingCloud IaaS -- *Backups*: None, all data can be recreated if necessary. - -[metad]:https://github.com/yunify/metad -[yunify]:https://github.com/yunify -[qingcloud]:https://qingcloud.com/ - - -## Yandex - -- *Application*: system configuration for services, service discovery -- *Launched*: March 2016 -- *Cluster Size*: 3 clusters of 5 members -- *Order of Data Size*: several gigabytes -- *Operator*: Yandex; [nekto0n][nekto0n] -- *Environment*: Bare Metal -- *Backups*: None - -[nekto0n]:https://github.com/nekto0n - -## Tencent Games - -- *Application*: Meta data and configuration data for service discovery, Kubernetes, etc. -- *Launched*: Jan. 2015 -- *Cluster Size*: 3 members each with 10s of clusters -- *Order of Data Size*: 10s of Megabytes -- *Operator*: Tencent Game Operations Department -- *Environment*: Baremetal -- *Backups*: Periodic sync to backup server - -In Tencent games, we use Docker and Kubernetes to deploy and run our applications, and use etcd to save meta data for service discovery, Kubernetes, etc. - -## Hyper.sh - -- *Application*: Kubernetes, distributed locks, etc. -- *Launched*: April 2016 -- *Cluster Size*: 1 cluster of 3 members -- *Order of Data Size*: 10s of MB -- *Operator*: Hyper.sh -- *Environment*: Baremetal -- *Backups*: None, all data can be recreated if necessary. - -In [hyper.sh][hyper.sh], the container service is backed by [hypernetes][hypernetes], a multi-tenant kubernetes distro. Moreover, we use etcd to coordinate the multiple manage services and store global meta data. - -[hypernetes]:https://github.com/hyperhq/hypernetes -[Hyper.sh]:https://www.hyper.sh - -## Meitu -- *Application*: system configuration for services, service discovery, kubernetes in test environment -- *Launched*: October 2015 -- *Cluster Size*: 1 cluster of 3 members -- *Order of Data Size*: megabytes -- *Operator*: Meitu, hxj@meitu.com, [shafreeck][shafreeck] -- *Environment*: Bare Metal -- *Backups*: None, all data can be recreated if necessary. - -[shafreeck]:https://github.com/shafreeck - -## Grab -- *Application*: system configuration for services, service discovery -- *Launched*: June 2016 -- *Cluster Size*: 1 cluster of 7 members -- *Order of Data Size*: megabytes -- *Operator*: Grab, [taxitan][taxitan], [reterVision][reterVision] -- *Environment*: AWS -- *Backups*: None, all data can be recreated if necessary. - -[taxitan]:https://github.com/taxitan -[reterVision]:https://github.com/reterVision - -## DaoCloud.io - -- *Application*: container management -- *Launched*: Sep. 2015 -- *Cluster Size*: 1000+ deployments, each deployment contains a 3 node cluster. -- *Order of Data Size*: 100s of Megabytes -- *Operator*: daocloud.io -- *Environment*: Baremetal and virtual machines -- *Backups*: None, all data can be recreated if necessary. - -In [DaoCloud][DaoCloud], we use Docker and Swarm to deploy and run our applications, and we use etcd to save metadata for service discovery. - -[DaoCloud]:https://www.daocloud.io - -## Branch.io - -- *Application*: Kubernetes -- *Launched*: April 2016 -- *Cluster Size*: Multiple clusters, multiple sizes -- *Order of Data Size*: 100s of Megabytes -- *Operator*: branch.io -- *Environment*: AWS, Kubernetes -- *Backups*: EBS volume backups - -At [Branch][branch], we use kubernetes heavily as our core microservice platform for staging and production. - -[branch]: https://branch.io - -## Baidu Waimai - -- *Application*: SkyDNS, Kubernetes, UDC, CMDB and other distributed systems -- *Launched*: April. 2016 -- *Cluster Size*: 3 clusters of 5 members -- *Order of Data Size*: several gigabytes -- *Operator*: Baidu Waimai Operations Department -- *Environment*: CentOS 6.5 -- *Backups*: backup scripts - -## Salesforce.com - -- *Application*: Kubernetes -- *Launched*: Jan 2017 -- *Cluster Size*: Multiple clusters of 3 members -- *Order of Data Size*: 100s of Megabytes -- *Operator*: Salesforce.com (krmayankk@github) -- *Environment*: BareMetal -- *Backups*: None, all data can be recreated - -## Hosted Graphite - -- *Application*: Service discovery, locking, ephemeral application data -- *Launched*: January 2017 -- *Cluster Size*: 2 clusters of 7 members -- *Order of Data Size*: Megabytes -- *Operator*: Hosted Graphite (sre@hostedgraphite.com) -- *Environment*: Bare Metal -- *Backups*: None, all data is considered ephemeral. - -## Transwarp - -- *Application*: Transwarp Data Cloud, Transwarp Operating System, Transwarp Data Hub, Sophon -- *Launched*: January 2016 -- *Cluster Size*: Multiple clusters, multiple sizes -- *Order of Data Size*: Megabytes -- *Operator*: Trasnwarp Operating System -- *Environment*: Bare Metal, Container -- *Backups*: backup scripts diff --git a/CHANGELOG-2.3.md b/CHANGELOG-2.3.md deleted file mode 100644 index 0b54062b1d86..000000000000 --- a/CHANGELOG-2.3.md +++ /dev/null @@ -1,16 +0,0 @@ - - -
- - -## [v2.3.8](https://github.com/etcd-io/etcd/releases/tag/v2.3.8) (2017-02-17) - -See [code changes](https://github.com/etcd-io/etcd/compare/v2.3.7...v2.3.8). - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md deleted file mode 100644 index bc11c80a5f0d..000000000000 --- a/CHANGELOG-3.0.md +++ /dev/null @@ -1,291 +0,0 @@ - - -
- - -## [v3.0.16](https://github.com/etcd-io/etcd/releases/tag/v3.0.16) (2016-11-13) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.15...v3.0.16) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.4*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.15](https://github.com/etcd-io/etcd/releases/tag/v3.0.15) (2016-11-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.14...v3.0.15) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Fixed - -- Fix cancel watch request with wrong range end. - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.14](https://github.com/etcd-io/etcd/releases/tag/v3.0.14) (2016-11-04) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.13...v3.0.14) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Added - -- v3 `etcdctl migrate` command now supports `--no-ttl` flag to discard keys on transform. - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.13](https://github.com/etcd-io/etcd/releases/tag/v3.0.13) (2016-10-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.12...v3.0.13) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.12](https://github.com/etcd-io/etcd/releases/tag/v3.0.12) (2016-10-07) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.11...v3.0.12) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.11](https://github.com/etcd-io/etcd/releases/tag/v3.0.11) (2016-10-07) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.10...v3.0.11) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Added - -- Server returns previous key-value (optional) - - `clientv3.WithPrevKV` option - - v3 etcdctl `put,watch,del --prev-kv` flag - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.10](https://github.com/etcd-io/etcd/releases/tag/v3.0.10) (2016-09-23) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.9...v3.0.10) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.9](https://github.com/etcd-io/etcd/releases/tag/v3.0.9) (2016-09-15) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.8...v3.0.9) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Added - -- Warn on domain names on listen URLs (v3.2 will reject domain names). - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.8](https://github.com/etcd-io/etcd/releases/tag/v3.0.8) (2016-09-09) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.7...v3.0.8) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Other - -- Allow only IP addresses in listen URLs (domain names are rejected). - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.7](https://github.com/etcd-io/etcd/releases/tag/v3.0.7) (2016-08-31) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.6...v3.0.7) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Other - -- SRV records only allow A records (RFC 2052). - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.6](https://github.com/etcd-io/etcd/releases/tag/v3.0.6) (2016-08-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.5...v3.0.6) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.5](https://github.com/etcd-io/etcd/releases/tag/v3.0.5) (2016-08-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.4...v3.0.5) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Other - -- SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given. - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.4](https://github.com/etcd-io/etcd/releases/tag/v3.0.4) (2016-07-27) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.3...v3.0.4) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Added - -- v2 `etcdctl ls` command now supports `--output=json`. -- Add /var/lib/etcd directory to etcd official Docker image. - -### Other - -- v2 auth can now use common name from TLS certificate when `--client-cert-auth` is enabled. - -### Go - -- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.3](https://github.com/etcd-io/etcd/releases/tag/v3.0.3) (2016-07-15) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.2...v3.0.3) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Other - -- Revert Dockerfile to use `CMD`, instead of `ENTRYPOINT`, to support `etcdctl` run. - - Docker commands for v3.0.2 won't work without specifying executable binary paths. -- v3 etcdctl default endpoints are now `127.0.0.1:2379`. - -### Go - -- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.2](https://github.com/etcd-io/etcd/releases/tag/v3.0.2) (2016-07-08) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.1...v3.0.2) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Other - -- Dockerfile uses `ENTRYPOINT`, instead of `CMD`, to run etcd without binary path specified. - -### Go - -- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.1](https://github.com/etcd-io/etcd/releases/tag/v3.0.1) (2016-07-01) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.0...v3.0.1) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6). - - -
- - -## [v3.0.0](https://github.com/etcd-io/etcd/releases/tag/v3.0.0) (2016-06-30) - -See [code changes](https://github.com/etcd-io/etcd/compare/v2.3.0...v3.0.0) and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_0/).** - -### Go - -- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6). - - -
- diff --git a/CHANGELOG-3.1.md b/CHANGELOG-3.1.md deleted file mode 100644 index 18765392ace9..000000000000 --- a/CHANGELOG-3.1.md +++ /dev/null @@ -1,578 +0,0 @@ - - -Previous change logs can be found at [CHANGELOG-3.0](https://github.com/etcd-io/etcd/blob/main/CHANGELOG-3.0.md). - - -The minimum recommended etcd versions to run in **production** are 3.1.11+, 3.2.26+, and 3.3.11+. - - -
- -## [v3.1.21](https://github.com/etcd-io/etcd/releases/tag/v3.1.21) (2019-TBD) - -### etcdctl v3 - -- [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2 -- Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). - - The command output is changed. Previously, if endpoint is unreachable, the command output is - "\ is unhealthy: failed to connect: \". This change unified the error message, all error types - now have the same output "\ is unhealthy: failed to commit proposal: \". - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). - -
- -## [v3.1.20](https://github.com/etcd-io/etcd/releases/tag/v3.1.20) (2018-10-10) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.19...v3.1.20) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Improved - -- Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. -- Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. -- Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. -- Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - - Previously, it only samples the TCP connection for snapshot messages. -- Display all registered [gRPC metrics at start](https://github.com/etcd-io/etcd/pull/10034). -- Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. -- Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. - -### client v3 - -- Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.19](https://github.com/etcd-io/etcd/releases/tag/v3.1.19) (2018-07-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.18...v3.1.19) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Improved - -- Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. -- Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. -- Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. -- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) Prometheus metric. - - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819). -- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - -### client v3 - -- Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.18](https://github.com/etcd-io/etcd/releases/tag/v3.1.18) (2018-06-15) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.17...v3.1.18) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.17](https://github.com/etcd-io/etcd/releases/tag/v3.1.17) (2018-06-06) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.16...v3.1.17) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Fix [v3 snapshot recovery](https://github.com/etcd-io/etcd/issues/7628). - - A follower receives a leader snapshot to be persisted as a `[SNAPSHOT-INDEX].snap.db` file on disk. - - Now, server [ensures that the incoming snapshot be persisted on disk before loading it](https://github.com/etcd-io/etcd/pull/7876). - - Otherwise, index mismatch happens and triggers server-side panic (e.g. newer WAL entry with outdated snapshot index). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.16](https://github.com/etcd-io/etcd/releases/tag/v3.1.16) (2018-05-31) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.15...v3.1.16) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - - Now, this server-side panic has been fixed. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.15](https://github.com/etcd-io/etcd/releases/tag/v3.1.15) (2018-05-09) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.14...v3.1.15) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Purge old [`*.snap.db` snapshot files](https://github.com/etcd-io/etcd/pull/7967). - - Previously, etcd did not respect `--max-snapshots` flag to purge old `*.snap.db` files. - - Now, etcd purges old `*.snap.db` files to keep maximum `--max-snapshots` number of files on disk. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.14](https://github.com/etcd-io/etcd/releases/tag/v3.1.14) (2018-04-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.13...v3.1.14) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. - -### etcd server - -- Add [`--initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - - By default, `--initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - - Now, this can be disabled by setting `--initial-election-tick-advance=false`. - - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `--initial-election-tick-advance` at the cost of slow initial bootstrap. - - If single-node, it advances ticks regardless. - - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.13](https://github.com/etcd-io/etcd/releases/tag/v3.1.13) (2018-03-29) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.12...v3.1.13) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Improved - -- Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/etcd-io/etcd/pull/9437). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.12](https://github.com/etcd-io/etcd/releases/tag/v3.1.12) (2018-03-08) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.11...v3.1.12) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9297). - - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.11](https://github.com/etcd-io/etcd/releases/tag/v3.1.11) (2017-11-28) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.10...v3.1.11) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- [#8411](https://github.com/etcd-io/etcd/issues/8411),[#8806](https://github.com/etcd-io/etcd/pull/8806) backport "mvcc: sending events after restore" -- [#8009](https://github.com/etcd-io/etcd/issues/8009),[#8902](https://github.com/etcd-io/etcd/pull/8902) backport coreos/bbolt v1.3.1-coreos.5 - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.1.10](https://github.com/etcd-io/etcd/releases/tag/v3.1.10) (2017-07-14) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.9...v3.1.10) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Added - -- Tag docker images with minor versions. - - e.g. `docker pull quay.io/coreos/etcd:v3.1` to fetch latest v3.1 versions. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - Fix panic on `net/http.CloseNotify` - - -
- - -## [v3.1.9](https://github.com/etcd-io/etcd/releases/tag/v3.1.9) (2017-06-09) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.8...v3.1.9) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Allow v2 snapshot over 512MB. - -### Go - -- Compile with [*Go 1.7.6*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.8](https://github.com/etcd-io/etcd/releases/tag/v3.1.8) (2017-05-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.7...v3.1.8) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.7](https://github.com/etcd-io/etcd/releases/tag/v3.1.7) (2017-04-28) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.6...v3.1.7) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.6](https://github.com/etcd-io/etcd/releases/tag/v3.1.6) (2017-04-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.5...v3.1.6) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Fill in Auth API response header. -- Remove auth check in Status API. - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.5](https://github.com/etcd-io/etcd/releases/tag/v3.1.5) (2017-03-27) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.4...v3.1.5) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd server - -- Fix raft memory leak issue. -- Fix Windows file path issues. - -### Other - -- Add `/etc/nsswitch.conf` file to alpine-based Docker image. - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.4](https://github.com/etcd-io/etcd/releases/tag/v3.1.4) (2017-03-22) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.3...v3.1.4) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.3](https://github.com/etcd-io/etcd/releases/tag/v3.1.3) (2017-03-10) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.2...v3.1.3) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd gateway - -- Fix `etcd gateway` schema handling in DNS discovery. -- Fix sd_notify behaviors in `gateway`, `grpc-proxy`. - -### gRPC Proxy - -- Fix sd_notify behaviors in `gateway`, `grpc-proxy`. - -### Other - -- Use machine default host when advertise URLs are default values(`localhost:2379,2380`) AND if listen URL is `0.0.0.0`. - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.2](https://github.com/etcd-io/etcd/releases/tag/v3.1.2) (2017-02-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.1...v3.1.2) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### etcd gateway - -- Fix `etcd gateway` with multiple endpoints. - -### Other - -- Use IPv4 default host, by default (when IPv4 and IPv6 are available). - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.1](https://github.com/etcd-io/etcd/releases/tag/v3.1.1) (2017-02-17) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.0...v3.1.1) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Go - -- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). - - -
- - -## [v3.1.0](https://github.com/etcd-io/etcd/releases/tag/v3.1.0) (2017-01-20) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.0...v3.1.0) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** - -### Improved - -- Faster linearizable reads (implements Raft [read-index](https://github.com/etcd-io/etcd/pull/6212)). -- v3 authentication API is now stable. - -### Breaking Changes - -- Deprecated following gRPC metrics in favor of [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus). - - `etcd_grpc_requests_total` - - `etcd_grpc_requests_failed_total` - - `etcd_grpc_active_streams` - - `etcd_grpc_unary_requests_duration_seconds` - -### Dependency - -- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`ugorji/go@9c7f9b7`**](https://github.com/ugorji/go/commit/9c7f9b7a2bc3a520f7c7b30b34b7f85f47fe27b6), and [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/6945). - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given. - - `TLSConfig.ServerName` is ignored with user-provided certificates for backwards compatibility; to be deprecated. - - For example, `etcd --discovery-srv=example.com` will only authenticate peers/clients when the provided certs have root domain `example.com` as an entry in Subject Alternative Name (SAN) field. - -### etcd server - -- Automatic leadership transfer when leader steps down. -- etcd flags - - `--strict-reconfig-check` flag is set by default. - - Add `--log-output` flag. - - Add `--metrics` flag. -- etcd uses default route IP if advertise URL is not given. -- Cluster rejects removing members if quorum will be lost. -- Discovery now has upper limit for waiting on retries. -- Warn on binding listeners through domain names; to be deprecated. -- v3.0 and v3.1 with `--auto-compaction-retention=10` run periodic compaction on v3 key-value store for every 10-hour. - - Compactor only supports periodic compaction. - - Compactor records latest revisions every 5-minute, until it reaches the first compaction period (e.g. 10-hour). - - In order to retain key-value history of last compaction period, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute. - - When `--auto-compaction-retention=10`, compactor uses revision 100 for compact revision where revision 100 is the latest revision fetched from 10 hours ago. - - If compaction succeeds or requested revision has already been compacted, it resets period timer and starts over with new historical revision records (e.g. restart revision collect and compact for the next 10-hour period). - - If compaction fails, it retries in 5 minutes. - -### client v3 - -- Add `SetEndpoints` method; update endpoints at runtime. -- Add `Sync` method; auto-update endpoints at runtime. -- Add `Lease TimeToLive` API; fetch lease information. -- replace Config.Logger field with global logger. -- Get API responses are sorted in ascending order by default. - -### etcdctl v3 - -- Add `lease timetolive` command. -- Add `--print-value-only` flag to get command. -- Add `--dest-prefix` flag to make-mirror command. -- `get` command responses are sorted in ascending order by default. - -### gRPC Proxy - -- Experimental gRPC proxy feature. - -### Other - -- `recipes` now conform to sessions defined in `clientv3/concurrency`. -- ACI has symlinks to `/usr/local/bin/etcd*`. - -### Go - -- Compile with [*Go 1.7.4*](https://golang.org/doc/devel/release.html#go1.7). - - -
- diff --git a/CHANGELOG-3.2.md b/CHANGELOG-3.2.md deleted file mode 100644 index b6b2d2a39cbb..000000000000 --- a/CHANGELOG-3.2.md +++ /dev/null @@ -1,1023 +0,0 @@ - - -Previous change logs can be found at [CHANGELOG-3.1](https://github.com/etcd-io/etcd/blob/main/CHANGELOG-3.1.md). - - -The minimum recommended etcd versions to run in **production** are 3.2.28+, 3.3.18+, and 3.4.2+. - -## v3.2.33 (TBD) - -
- -## [v3.2.32](https://github.com/etcd-io/etcd/releases/tag/v3.2.32) (2021-03-28) -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.31...v3.2.32) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -### Package `wal` -- add wal slice bound check to make sure entry index is not greater than the number of entries -- check slice size in decodeRecord -- fix panic when decoder not set - -### Package `fileutil` -- fix constant for linux locking - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.2.31](https://github.com/etcd-io/etcd/releases/tag/v3.2.31) (2020-08-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.30...v3.2.31) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -### auth, etcdserver - -- Improve [`runtime.FDUsage` call pattern to reduce objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). -- [attaching a fake root token when calling `LeaseRevoke`](https://github.com/etcd-io/etcd/pull/11691). - - fix a data corruption bug caused by lease expiration when authentication is enabled and upgrading cluster from etcd-3.2 to etcd-3.3 - -### Package `runtime` - -- Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). - -### Metrics, Monitoring - -- Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.2.30](https://github.com/etcd-io/etcd/releases/tag/v3.2.30) (2020-04-01) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.29...v3.2.30) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -### Package `wal` - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - -### Metrics, Monitoring - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.2.29](https://github.com/etcd-io/etcd/releases/tag/v3.2.29) (2020-03-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.28...v3.2.29) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -### etcd server - -- [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). -- Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). - -### client v3 - -- Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -- Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.28](https://github.com/etcd-io/etcd/releases/tag/v3.2.28) (2019-11-10) - -### Improved - -- Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/11195). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11271) Prometheus metric. - -### etcdserver - -- Fix [`wait purge file loop during shutdown`](https://github.com/etcd-io/etcd/pull/11308). - - Previously, during shutdown etcd could accidentally remove needed wal files, resulting in catastrophic error `etcdserver: open wal error: wal: file not found.` during startup. - - Now, etcd makes sure the purge file loop exits before server signals stop of the raft node. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.27](https://github.com/etcd-io/etcd/releases/tag/v3.2.27) (2019-09-17) - -### etcdctl v3 - -- [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2 -- Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). - - The command output is changed. Previously, if endpoint is unreachable, the command output is - "\ is unhealthy: failed to connect: \". This change unified the error message, all error types - now have the same output "\ is unhealthy: failed to commit proposal: \". -- Fix [`etcdctl snapshot status` to not modify snapshot file](https://github.com/etcd-io/etcd/pull/11157). - - For example, start etcd `v3.3.10` - - Write some data - - Use etcdctl `v3.3.10` to save snapshot - - Somehow, upgrading Kubernetes fails, thus rolling back to previous version etcd `v3.2.24` - - Run etcdctl `v3.2.24` `snapshot status` against the snapshot file saved from `v3.3.10` server - - Run etcdctl `v3.2.24` `snapshot restore` fails with `"expected sha256 [12..."` - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). -- Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. -- Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.26](https://github.com/etcd-io/etcd/releases/tag/v3.2.26) (2019-01-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.25...v3.2.26) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### gRPC Proxy - -- Fix [memory leak in cache layer](https://github.com/etcd-io/etcd/pull/10327). - -### Security, Authentication - -- Disable [CommonName authentication for gRPC-gateway](https://github.com/etcd-io/etcd/pull/10366) gRPC-gateway proxy requests to etcd server use the etcd client server TLS certificate. If that certificate contains CommonName we do not want to use that for authentication as it could lead to permission escalation. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.25](https://github.com/etcd-io/etcd/releases/tag/v3.2.25) (2018-10-10) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.24...v3.2.25) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. -- Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. -- Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. -- Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - - Previously, it only samples the TCP connection for snapshot messages. -- Display all registered [gRPC metrics at start](https://github.com/etcd-io/etcd/pull/10032). -- Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. -- Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. - -### client v3 - -- Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.24](https://github.com/etcd-io/etcd/releases/tag/v3.2.24) (2018-07-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.23...v3.2.24) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. -- Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. -- Add [`etcd_server_slow_apply_total`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. -- Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. -- Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. -- Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. -- Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. -- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) Prometheus metric. - - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819). -- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_server_quota_backend_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - -### gRPC Proxy - -- Add [flags for specifying TLS for connecting to proxy](https://github.com/etcd-io/etcd/pull/9894): - - Add `grpc-proxy start --cert-file`, `grpc-proxy start --key-file` and `grpc-proxy start --trusted-ca-file` flags. -- Add [`grpc-proxy start --metrics-addr` flag for specifying a separate metrics listen address](https://github.com/etcd-io/etcd/pull/9894). - -### client v3 - -- Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.23](https://github.com/etcd-io/etcd/releases/tag/v3.2.23) (2018-06-15) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.22...v3.2.23) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Improve [slow request apply warning log](https://github.com/etcd-io/etcd/pull/9288). - - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`. - - Redact [request value field](https://github.com/etcd-io/etcd/pull/9822). - - Provide [response size](https://github.com/etcd-io/etcd/pull/9826). -- Add [backoff on watch retries on transient errors](https://github.com/etcd-io/etcd/pull/9840). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.22](https://github.com/etcd-io/etcd/releases/tag/v3.2.22) (2018-06-06) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.21...v3.2.22) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Security, Authentication - -- Support TLS cipher suite whitelisting. - - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - - TLS handshake fails when client hello is requested with invalid cipher suites. - - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - - If empty, Go auto-populates the list. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.21](https://github.com/etcd-io/etcd/releases/tag/v3.2.21) (2018-05-31) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.20...v3.2.21) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Fix [auth storage panic when simple token provider is disabled](https://github.com/etcd-io/etcd/pull/8695). -- Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - - Now, this server-side panic has been fixed. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.20](https://github.com/etcd-io/etcd/releases/tag/v3.2.20) (2018-05-09) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.19...v3.2.20) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Purge old [`*.snap.db` snapshot files](https://github.com/etcd-io/etcd/pull/7967). - - Previously, etcd did not respect `--max-snapshots` flag to purge old `*.snap.db` files. - - Now, etcd purges old `*.snap.db` files to keep maximum `--max-snapshots` number of files on disk. - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.19](https://github.com/etcd-io/etcd/releases/tag/v3.2.19) (2018-04-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.18...v3.2.19) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/9557) Prometheus metric. -- Fix [race conditions in v2 server stat collecting](https://github.com/etcd-io/etcd/pull/9562). -- Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. - -### Security, Authentication - -- Fix [TLS reload](https://github.com/etcd-io/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/etcd-io/etcd/issues/9541). - - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets. - - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/etcd-io/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online. - - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs). - -### etcd server - -- Add [`etcd --initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - - Now, this can be disabled by setting `--initial-election-tick-advance=false`. - - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `--initial-election-tick-advance` at the cost of slow initial bootstrap. - - If single-node, it advances ticks regardless. - - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.18](https://github.com/etcd-io/etcd/releases/tag/v3.2.18) (2018-03-29) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.17...v3.2.18) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/etcd-io/etcd/pull/9437). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.17](https://github.com/etcd-io/etcd/releases/tag/v3.2.17) (2018-03-08) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.16...v3.2.17) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/etcd-io/etcd/pull/9379). - - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server. - - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`. -- Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/etcd-io/etcd/pull/9399). - - `TTL` parameter to `Grant` request is unit of second. - - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/etcd-io/etcd/issues/9374). - - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! -- Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/etcd-io/etcd/pull/9347). - -### Proxy v2 - -- Fix [v2 proxy leaky HTTP requests](https://github.com/etcd-io/etcd/pull/9336). - -### Go - -- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.16](https://github.com/etcd-io/etcd/releases/tag/v3.2.16) (2018-02-12) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.15...v3.2.16) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9297). - - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.15](https://github.com/etcd-io/etcd/releases/tag/v3.2.15) (2018-01-22) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.14...v3.2.15) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Prevent [server panic from member update/add](https://github.com/etcd-io/etcd/pull/9174) with [wrong scheme URLs](https://github.com/etcd-io/etcd/issues/9173). -- Log [user context cancel errors on stream APIs in debug level with TLS](https://github.com/etcd-io/etcd/pull/9178). - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.14](https://github.com/etcd-io/etcd/releases/tag/v3.2.14) (2018-01-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.13...v3.2.14) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Log [user context cancel errors on stream APIs in debug level](https://github.com/etcd-io/etcd/pull/9105). - -### etcd server - -- Fix [`mvcc/backend.defragdb` nil-pointer dereference on create bucket failure](https://github.com/etcd-io/etcd/pull/9119). - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.13](https://github.com/etcd-io/etcd/releases/tag/v3.2.13) (2018-01-02) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.12...v3.2.13) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Remove [verbose error messages on stream cancel and gRPC info-level logs](https://github.com/etcd-io/etcd/pull/9080) in server-side. -- Fix [gRPC server panic on `GracefulStop` TLS-enabled server](https://github.com/etcd-io/etcd/pull/8987). - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.12](https://github.com/etcd-io/etcd/releases/tag/v3.2.12) (2017-12-20) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.11...v3.2.12) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Dependency - -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) to [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5). -- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.3`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3) to [**`v1.3.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0). - -### etcd server - -- Fix [error message of `Revision` compactor](https://github.com/etcd-io/etcd/pull/8999) in server-side. - -### client v3 - -- Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/etcd-io/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/etcd-io/etcd/clientv3#Config). - - Fix [exceeded response size limit error in client-side](https://github.com/etcd-io/etcd/issues/9043). - - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099). - - In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB. - - `MaxCallSendMsgSize` default value is 2 MiB, if not configured. - - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured. - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.11](https://github.com/etcd-io/etcd/releases/tag/v3.2.11) (2017-12-05) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.10...v3.2.11) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Dependency - -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.7.3`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.3) to [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4). - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- Log [more details on TLS handshake failures](https://github.com/etcd-io/etcd/pull/8952/files). - -### client v3 - -- Fix racey grpc-go's server handler transport `WriteStatus` call to prevent [TLS-enabled etcd server crash](https://github.com/etcd-io/etcd/issues/8904). -- Add [gRPC RPC failure warnings](https://github.com/etcd-io/etcd/pull/8939) to help debug such issues in the future. - -### Documentation - -- Remove `--listen-metrics-urls` flag in monitoring document (non-released in `v3.2.x`, planned for `v3.3.x`). - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.10](https://github.com/etcd-io/etcd/releases/tag/v3.2.10) (2017-11-16) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.9...v3.2.10) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Dependency - -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1) to [**`v1.7.3`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.3). -- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.2.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.0) to [**`v1.3`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3). - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- Revert [discovery SRV auth `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/etcd-io/etcd/pull/8651) to support non-wildcard subject alternative names in the certs (see [issue #8445](https://github.com/etcd-io/etcd/issues/8445) for more contexts). - - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `etcd.local` (**not `*.etcd.local`**) as an entry in Subject Alternative Name (SAN) field. - -### etcd server - -- Replace backend key-value database `boltdb/bolt` with [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to address [backend database size issue](https://github.com/etcd-io/etcd/issues/8009). - -### client v3 - -- Rewrite balancer to handle [network partitions](https://github.com/etcd-io/etcd/issues/8711). - -### Go - -- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.9](https://github.com/etcd-io/etcd/releases/tag/v3.2.9) (2017-10-06) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.8...v3.2.9) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- Update `golang.org/x/crypto/bcrypt` (see [golang/crypto@6c586e1](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117)). -- Fix discovery SRV bootstrapping to [authenticate `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/etcd-io/etcd/pull/8651), in order to support sub-domain wildcard matching (see [issue #8445](https://github.com/etcd-io/etcd/issues/8445) for more contexts). - - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `*.etcd.local` as an entry in Subject Alternative Name (SAN) field. - -### Go - -- Compile with [*Go 1.8.4*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.8](https://github.com/etcd-io/etcd/releases/tag/v3.2.8) (2017-09-29) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.7...v3.2.8) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### client v2 - -- Fix v2 client failover to next endpoint on mutable operation. - -### gRPC Proxy - -- Handle [`KeysOnly` flag](https://github.com/etcd-io/etcd/pull/8552). - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.7](https://github.com/etcd-io/etcd/releases/tag/v3.2.7) (2017-09-01) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.6...v3.2.7) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Security, Authentication - -- Fix [server-side auth so concurrent auth operations do not return old revision error](https://github.com/etcd-io/etcd/pull/8306). - -### client v3 - -- Fix [`concurrency/stm` Put with serializable snapshot](https://github.com/etcd-io/etcd/pull/8439). - - Use store revision from first fetch to resolve write conflicts instead of modified revision. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.6](https://github.com/etcd-io/etcd/releases/tag/v3.2.6) (2017-08-21) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.5...v3.2.6) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Fix watch restore from snapshot. -- Fix multiple URLs for `--listen-peer-urls` flag. -- Add `--enable-pprof` flag to etcd configuration file format. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix `etcd_debugging_mvcc_keys_total` inconsistency. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.5](https://github.com/etcd-io/etcd/releases/tag/v3.2.5) (2017-08-04) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.4...v3.2.5) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcdctl v3 - -- Return non-zero exit code on unhealthy `endpoint health`. - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/etcd-io/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) SAN field is `["*.example.default.svc", "*.example.default.svc.cluster.local"]` when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A reverse-lookup the IP `10.138.0.2` to get the list of host names. And either exact or wildcard match the host names with peer B's cert DNS names in Subject Alternative Name (SAN) field. If none of reverse/forward lookups worked, it returns an error `"tls: "10.138.0.2" does not match any of DNSNames ["*.example.default.svc","*.example.default.svc.cluster.local"]`. See [issue#8268](https://github.com/etcd-io/etcd/issues/8268) for more detail. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix unreachable `/metrics` endpoint when `--enable-v2=false`. - -### gRPC Proxy - -- Handle [`PrevKv` flag](https://github.com/etcd-io/etcd/pull/8366). - -### Other - -- Add container registry `gcr.io/etcd-development/etcd`. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.4](https://github.com/etcd-io/etcd/releases/tag/v3.2.4) (2017-07-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.3...v3.2.4) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Do not block on active client stream when stopping server - -### gRPC proxy - -- Fix gRPC proxy Snapshot RPC error handling - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.3](https://github.com/etcd-io/etcd/releases/tag/v3.2.3) (2017-07-14) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.2...v3.2.3) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### client v3 - -- Let clients establish unlimited streams - -### Other - -- Tag docker images with minor versions - - e.g. `docker pull quay.io/coreos/etcd:v3.2` to fetch latest v3.2 versions - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.2](https://github.com/etcd-io/etcd/releases/tag/v3.2.2) (2017-07-07) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.1...v3.2.2) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Rate-limit lease revoke on expiration. -- Extend leases on promote to avoid queueing effect on lease expiration. - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- [Server accepts connections if IP matches, without checking DNS entries](https://github.com/etcd-io/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) SAN field is `["invalid.domain", "10.138.0.2"]` when peer B's remote IP address is `10.138.0.2` and `invalid.domain` is a invalid host. When peer B tries to join the cluster, peer A successfully authenticates B, since Subject Alternative Name (SAN) field has a valid matching IP address. See [issue#8206](https://github.com/etcd-io/etcd/issues/8206) for more detail. - -### etcd server - -- Accept connection with matched IP SAN but no DNS match. - - Don't check DNS entries in certs if there's a matching IP. - -### gRPC gateway - -- Use user-provided listen address to connect to gRPC gateway. - - `net.Listener` rewrites IPv4 0.0.0.0 to IPv6 [::], breaking IPv6 disabled hosts. - - Only v3.2.0, v3.2.1 are affected. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.1](https://github.com/etcd-io/etcd/releases/tag/v3.2.1) (2017-06-23) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.0...v3.2.1) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### etcd server - -- Fix backend database in-memory index corruption issue on restore (only 3.2.0 is affected). - -### gRPC gateway - -- Fix Txn marshaling. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix backend database size debugging metrics. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- - -## [v3.2.0](https://github.com/etcd-io/etcd/releases/tag/v3.2.0) (2017-06-09) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.0...v3.2.0) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** - -### Improved - -- Improve backend read concurrency. - -### Breaking Changes - -- Increased [`--snapshot-count` default value from 10,000 to 100,000](https://github.com/etcd-io/etcd/pull/7160). - - Higher snapshot count means it holds Raft entries in memory for longer before discarding old entries. - - It is a trade-off between less frequent snapshotting and [higher memory usage](https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-371977156). - - User lower `--snapshot-count` value for lower memory usage. - - User higher `--snapshot-count` value for better availabilities of slow followers (less frequent snapshots from leader). -- `clientv3.Lease.TimeToLive` returns `LeaseTimeToLiveResponse.TTL == -1` on lease not found. -- `clientv3.NewFromConfigFile` is moved to `clientv3/yaml.NewConfig`. -- `embed.Etcd.Peers` field is now `[]*peerListener`. -- Rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since [domain name is invalid for network interface binding](https://github.com/etcd-io/etcd/issues/6336). - -### Dependency - -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.0.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.0.4) to [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1). -- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) to [**`v1.2.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.0). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_disk_backend_snapshot_duration_seconds`](https://github.com/etcd-io/etcd/pull/7892) -- Add `etcd_debugging_server_lease_expired_total` metrics. - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- [TLS certificates get reloaded on every client connection](https://github.com/etcd-io/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/etcd-io/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757). -- [Server denies incoming peer certs with wrong IP `SAN`](https://github.com/etcd-io/etcd/pull/7687). For instance, if peer cert contains any IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster. For example, peer B's CSR (with `cfssl`) SAN field is `["*.example.default.svc", "*.example.default.svc.cluster.local", "10.138.0.27"]` when peer B's actual IP address is `10.138.0.2`, not `10.138.0.27`. When peer B tries to join the cluster, peer A will reject B with the error `x509: certificate is valid for 10.138.0.27, not 10.138.0.2`, because B's remote IP address does not match the one in Subject Alternative Name (SAN) field. -- [Server resolves TLS `DNSNames` when checking `SAN`](https://github.com/etcd-io/etcd/pull/7767). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups (`dig b.com`) on those DNS names have matching IP with the remote IP address. For example, peer B's CSR (with `cfssl`) SAN field is `["b.com"]` when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A looks up the incoming host `b.com` to get the list of IP addresses (e.g. `dig b.com`). And rejects B if the list does not contain the IP `10.138.0.2`, with the error `tls: 10.138.0.2 does not match any of DNSNames ["b.com"]`. -- Auth support JWT token. - -### etcd server - -- RPCs - - Add Election, Lock service. -- Native client `etcdserver/api/v3client` - - client "embedded" in the server. -- Logging, monitoring - - Server warns large snapshot operations. -- Add `etcd --enable-v2` flag to enable v2 API server. - - `etcd --enable-v2=true` by default. -- Add `etcd --auth-token` flag. -- v3.2 compactor runs [every hour](https://github.com/etcd-io/etcd/pull/7875). - - Compactor only supports periodic compaction. - - Compactor continues to record latest revisions every 5-minute. - - For every hour, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute. - - That is, for every hour, compactor discards historical data created before compaction period. - - The retention window of compaction period moves to next hour. - - For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2 compacts revision 1000, 1100, and 1200 for every 1-hour. - - If compaction succeeds or requested revision has already been compacted, it resets period timer and removes used compacted revision from historical revision records (e.g. start next revision collect and compaction from previously collected revisions). - - If compaction fails, it retries in 5 minutes. -- Allow snapshot over 512MB. - -### client v3 - -- STM prefetching. -- Add namespace feature. -- Add `ErrOldCluster` with server version checking. -- Translate `WithPrefix()` into `WithFromKey()` for empty key. - -### etcdctl v3 - -- Add `check perf` command. -- Add `etcdctl --from-key` flag to role grant-permission command. -- `lock` command takes an optional command to execute. - -### gRPC Proxy - -- Proxy endpoint discovery. -- Namespaces. -- Coalesce lease requests. - -### etcd gateway - -- Support [DNS SRV priority](https://github.com/etcd-io/etcd/pull/7882) for [smart proxy routing](https://github.com/etcd-io/etcd/issues/4378). - -### Other - -- v3 client - - concurrency package's elections updated to match RPC interfaces. - - let client dial endpoints not in the balancer. -- Release - - Annotate acbuild with supports-systemd-notify. - - Add `nsswitch.conf` to Docker container image. - - Add ppc64le, arm64(experimental) builds. - -### Go - -- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - - -
- diff --git a/CHANGELOG-3.3.md b/CHANGELOG-3.3.md deleted file mode 100644 index 1f914fad7769..000000000000 --- a/CHANGELOG-3.3.md +++ /dev/null @@ -1,1094 +0,0 @@ - - -Previous change logs can be found at [CHANGELOG-3.2](https://github.com/etcd-io/etcd/blob/main/CHANGELOG-3.2.md). - - -The minimum recommended etcd versions to run in **production** are 3.2.28+, 3.3.18+, and 3.4.2+. - - -
- - -## v3.3.25 (2020 TBD) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.23...v3.3.25) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### Security - -- A [log warning](https://github.com/etcd-io/etcd/pull/12242) is added when etcd use any existing directory that has a permission different than 700 on Linux and 777 on Windows. - - -## [v3.3.24](https://github.com/etcd-io/etcd/releases/tag/v3.3.24) (2020-08-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.23...v3.3.24) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### Package `etcd server` - -- Fix [`int64` convert panic in raft logger](https://github.com/etcd-io/etcd/pull/12106). - - Fix [kubernetes/kubernetes#91937](https://github.com/kubernetes/kubernetes/issues/91937). - -### Package `runtime` - -- Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). - -### Metrics, Monitoring - -- Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - - -
- - - -## [v3.3.23](https://github.com/etcd-io/etcd/releases/tag/v3.3.23) (2020-07-16) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.22...v3.3.23) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### Breaking Changes - -- Fix [incorrect package dependency when etcd clientv3 used as libary](https://github.com/etcd-io/etcd/issues/12068). -- Changed behavior on [existing dir permission](https://github.com/etcd-io/etcd/pull/11798). - - Previously, the permission was not checked on existing data directory and the directory used for automatically generating self-signed certificates for TLS connections with clients. Now a check is added to make sure those directories, if already exist, has a desired permission of 700 on Linux and 777 on Windows. - -### Package `wal` - -### etcd server -- Fix [watch stream got closed if one watch request is not permitted](https://github.com/etcd-io/etcd/pull/11758). -- Add [etcd --auth-token-ttl](https://github.com/etcd-io/etcd/pull/11980) flag to customize `simpleTokenTTL` settings. -- Improve [runtime.FDUsage objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). -- Improve [mvcc.watchResponse channel Memory Usage](https://github.com/etcd-io/etcd/pull/11987). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - - -
- - -## [v3.3.22](https://github.com/etcd-io/etcd/releases/tag/v3.3.22) (2020-05-20) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.21...v3.3.22) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### Package `wal` - -- Add [missing CRC checksum check in WAL validate method otherwise causes panic](https://github.com/etcd-io/etcd/pull/11924). - - See https://github.com/etcd-io/etcd/issues/11918. - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.3.21](https://github.com/etcd-io/etcd/releases/tag/v3.3.21) (2020-05-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.20...v3.3.21) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### `etcdctl` - -- Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). - -### Package `clientv3` - -- Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). - -### etcd server - -- Improve logging around snapshot send and receive. -- [Add log when etcdserver failed to apply command](https://github.com/etcd-io/etcd/pull/11670). -- [Fix deadlock bug in mvcc](https://github.com/etcd-io/etcd/pull/11817). -- Fix [inconsistency between WAL and server snapshot](https://github.com/etcd-io/etcd/pull/11888). - - Previously, server restore fails if it had crashed after persisting raft hard state but before saving snapshot. - - See https://github.com/etcd-io/etcd/issues/10219 for more. - -### Package `auth` - -- [Fix a data corruption bug by saving consistent index](https://github.com/etcd-io/etcd/pull/11652). - -### Metrics, Monitoring - -- Add [`etcd_debugging_auth_revision`](https://github.com/etcd-io/etcd/commit/f14d2a087f7b0fd6f7980b95b5e0b945109c95f3). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.3.20](https://github.com/etcd-io/etcd/releases/tag/v3.3.20) (2020-04-01) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.19...v3.3.20) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### Package `wal` - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - -### Metrics, Monitoring - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.3.19](https://github.com/etcd-io/etcd/releases/tag/v3.3.19) (2020-03-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.18...v3.3.19) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### client v3 - -- Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. - -### etcd server - -- [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). -- Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). - -### etcdctl v3 - -- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11638) command to prevent potential timeout. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -- Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). - -### gRPC Proxy - -- Fix [`panic on error`](https://github.com/etcd-io/etcd/pull/11694) for metrics handler. - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.3.18](https://github.com/etcd-io/etcd/releases/tag/v3.3.18) (2019-11-26) - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11261) Prometheus metric. -- Add [`etcd_debugging_mvcc_total_put_size_in_bytes`](https://github.com/etcd-io/etcd/pull/11374) Prometheus metric. - -### etcdserver - -- Fix [`wait purge file loop during shutdown`](https://github.com/etcd-io/etcd/pull/11308). - - Previously, during shutdown etcd could accidentally remove needed wal files, resulting in catastrophic error `etcdserver: open wal error: wal: file not found.` during startup. - - Now, etcd makes sure the purge file loop exits before server signals stop of the raft node. - - -
- - -## [v3.3.17](https://github.com/etcd-io/etcd/releases/tag/v3.3.17) (2019-10-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.16...v3.3.17) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -### Release details - -This release replaces 3.3.16. - -Due to the etcd 3.3.16 release being incorrectly released (see details below), please use this release instead. - - -
- - -## [v3.3.16](https://github.com/etcd-io/etcd/releases/tag/v3.3.16) (2019-10-10) - -**WARNING: This is a bad release! Please use etcd 3.3.17 instead. See https://github.com/etcd-io/etcd/issues/11241 for details.** - -### Issues with release - -- go mod for 'v3.3.16' may return a different hash if retrieved from a go mod proxy than if retrieved directly from github. Depending on this version is unsafe. See https://github.com/etcd-io/etcd/issues/11241 for details. -- The binaries and docker image for this release have been published and will be left as-is, but will not be signed since this is a bad release. - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.15...v3.3.16) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/11196). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. -- Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - -### Dependency - -- Upgrade [`github.com/coreos/bbolt`](https://github.com/etcd-io/bbolt/releases) from [**`v1.3.1-coreos.6`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.1-coreos.6) to [**`v1.3.3`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3). - -### etcdctl v3 - -- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout. - -### Go - -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - -### client v3 - -- Fix [client balancer failover against multiple endpoints](https://github.com/etcd-io/etcd/pull/11184). - - Fix ["kube-apiserver: failover on multi-member etcd cluster fails certificate check on DNS mismatch" (kubernetes#83028)](https://github.com/kubernetes/kubernetes/issues/83028). -- Fix [IPv6 endpoint parsing in client](https://github.com/etcd-io/etcd/pull/11211). - - Fix ["1.16: etcd client does not parse IPv6 addresses correctly when members are joining" (kubernetes#83550)](https://github.com/kubernetes/kubernetes/issues/83550). - - -
- - -## [v3.3.15](https://github.com/etcd-io/etcd/releases/tag/v3.3.15) (2019-08-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.14...v3.3.15) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -NOTE: This patch release had to include some new features from 3.4, while trying to minimize the difference between client balancer implementation. This release fixes ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - -### Breaking Changes - -- Revert "Migrate dependency management tool from `glide` to [Go module](https://github.com/etcd-io/etcd/pull/10063)". - - Now, etcd >= v3.3.15 uses `glide` for dependency management. - - See [kubernetes#81434](https://github.com/kubernetes/kubernetes/pull/81434) for more contexts. - -### Go - -- Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - - -
- - -## [v3.3.14](https://github.com/etcd-io/etcd/releases/tag/v3.3.14) (2019-08-16) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.13...v3.3.14) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -- [v3.3.14-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.14-rc.0) (2019-08-15), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.14-beta.0...v3.3.14-rc.0). -- [v3.3.14-beta.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.14-beta.0) (2019-08-14), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.13...v3.3.14-beta.0). - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -NOTE: This patch release had to include some new features from 3.4, while trying to minimize the difference between client balancer implementation. This release fixes ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - -### Breaking Changes - -- Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. to block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. -- Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. -- Migrate dependency management tool from `glide` to [Go module](https://github.com/etcd-io/etcd/pull/10063). - - <= 3.3 puts `vendor` directory under `cmd/vendor` directory to [prevent conflicting transitive dependencies](https://github.com/etcd-io/etcd/issues/4913). - - 3.4 moves `cmd/vendor` directory to `vendor` at repository root. - - Remove recursive symlinks in `cmd` directory. - - Now `go get/install/build` on `etcd` packages (e.g. `clientv3`, `tools/benchmark`) enforce builds with etcd `vendor` directory. -- Deprecated `latest` [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tag. - - **`docker pull gcr.io/etcd-development/etcd:latest` would not be up-to-date**. -- Deprecated [minor](https://semver.org/) version [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tags. - - `docker pull gcr.io/etcd-development/etcd:v3.3` would still work but may be stale. - - **`docker pull gcr.io/etcd-development/etcd:v3.4` would not work**. - - Use **`docker pull gcr.io/etcd-development/etcd:v3.3.14`** instead, with the exact patch version. -- Deprecated [ACIs from official release](https://github.com/etcd-io/etcd/pull/9059). - - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016. - - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore. - - `*.aci` files are not available from `v3.4` release. - -### etcd server - -- Add [`rpctypes.ErrLeaderChanged`](https://github.com/etcd-io/etcd/pull/10094). - - Now linearizable requests with read index would fail fast when there is a leadership change, instead of waiting until context timeout. -- Fix [race condition in `rafthttp` transport pause/resume](https://github.com/etcd-io/etcd/pull/10826). - -### API - -- Add [`watch_id` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9065) to allow user-provided watch ID to `mvcc`. - - Corresponding `watch_id` is returned via `etcdserverpb.WatchResponse`, if any. -- Add [`fragment` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9291) to request etcd server to [split watch events](https://github.com/etcd-io/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes. - - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes. - - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit. - - Useful when client-side has limited bandwidths. - - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client. - - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`. - - Client must implement fragmented watch event merge (which `clientv3` does in etcd v3.4). -- Add [`WatchRequest.WatchProgressRequest`](https://github.com/etcd-io/etcd/pull/9869). - - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - - Think of it as `WithProgressNotify` that can be triggered manually. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_network_snapshot_send_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. -- Add [`etcd_network_snapshot_receive_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. -- Add [`etcd_server_snapshot_apply_in_progress_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. - -### client v3 - -- Fix [gRPC panic "send on closed channel](https://github.com/etcd-io/etcd/issues/9956) by upgrading [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5) to [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0). -- Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. to block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. - -### etcdctl v3 - -- Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). - - The command output is changed. Previously, if endpoint is unreachable, the command output is - "\ is unhealthy: failed to connect: \". This change unified the error message, all error types - now have the same output "\ is unhealthy: failed to commit proposal: \". -- Add [missing newline in `etcdctl endpoint health`](https://github.com/etcd-io/etcd/pull/10793). - -### Package `pkg/adt` - -- Change [`pkg/adt.IntervalTree` from `struct` to `interface`](https://github.com/etcd-io/etcd/pull/10959). - - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). -- Improve [`pkg/adt.IntervalTree` test coverage](https://github.com/etcd-io/etcd/pull/10959). - - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). -- Fix [Red-Black tree to maintain black-height property](https://github.com/etcd-io/etcd/pull/10978). - - Previously, delete operation violates [black-height property](https://github.com/etcd-io/etcd/issues/10965). - -### Go - -- Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - - -
- - -## [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) (2019-05-02) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.12...v3.3.13) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Improve [heartbeat send failure logging](https://github.com/etcd-io/etcd/pull/10663). -- Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). - -### client v3 - -- Fix [`(*Client).Endpoints()` method race condition](https://github.com/etcd-io/etcd/pull/10595). - -### Package `wal` - -- Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). - -### Dependency - -- Migrate [`github.com/ugorji/go/codec`](https://github.com/ugorji/go/releases) to [**`github.com/json-iterator/go`**](https://github.com/json-iterator/go) (See [#10667](https://github.com/etcd-io/etcd/pull/10667) for more). -- Migrate [`github.com/ghodss/yaml`](https://github.com/ghodss/yaml/releases) to [**`sigs.k8s.io/yaml`**](https://github.com/kubernetes-sigs/yaml) (See [#10718](https://github.com/etcd-io/etcd/pull/10718) for more). - -### Go - -- Compile with [*Go 1.10.8*](https://golang.org/doc/devel/release.html#go1.10). - - -
- - -## [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) (2019-02-07) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.11...v3.3.12) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### etcdctl v3 - -- [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2 - -### Go - -- Compile with [*Go 1.10.8*](https://golang.org/doc/devel/release.html#go1.10). - - -
- - -## [v3.3.11](https://github.com/etcd-io/etcd/releases/tag/v3.3.11) (2019-01-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.10...v3.3.11) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### gRPC Proxy - -- Fix [memory leak in cache layer](https://github.com/etcd-io/etcd/pull/10327). - -### Security, Authentication - -- Disable [CommonName authentication for gRPC-gateway](https://github.com/etcd-io/etcd/pull/10366) gRPC-gateway proxy requests to etcd server use the etcd client server TLS certificate. If that certificate contains CommonName we do not want to use that for authentication as it could lead to permission escalation. - -### Go - -- Compile with [*Go 1.10.7*](https://golang.org/doc/devel/release.html#go1.10). - - -
- - -## [v3.3.10](https://github.com/etcd-io/etcd/releases/tag/v3.3.10) (2018-10-10) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.9...v3.3.10) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. -- Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. -- Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. -- Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - - Previously, it only samples the TCP connection for snapshot messages. -- Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. -- Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. - -### client v3 - -- Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. - -### Go - -- Compile with [*Go 1.10.4*](https://golang.org/doc/devel/release.html#go1.10). - - -
- - -## [v3.3.9](https://github.com/etcd-io/etcd/releases/tag/v3.3.9) (2018-07-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.8...v3.3.9) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). - -### Security, Authentication - -- Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10) to support [crypto/x509 "Name Constraints"](https://github.com/etcd-io/etcd/issues/9912). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. -- Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. -- Add [`etcd_server_slow_apply_total`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. -- Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. -- Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. -- Add [`etcd_mvcc_hash_rev_duration_seconds`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. -- Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. -- Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. -- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) Prometheus metric. - - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819). -- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - -### client v3 - -- Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. - -### Go - -- Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10). - - -
- - -## [v3.3.8](https://github.com/etcd-io/etcd/releases/tag/v3.3.8) (2018-06-15) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.7...v3.3.8) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Improve [slow request apply warning log](https://github.com/etcd-io/etcd/pull/9288). - - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`. - - Redact [request value field](https://github.com/etcd-io/etcd/pull/9822). - - Provide [response size](https://github.com/etcd-io/etcd/pull/9826). -- Add [backoff on watch retries on transient errors](https://github.com/etcd-io/etcd/pull/9840). - -### Go - -- Compile with [*Go 1.9.7*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.7](https://github.com/etcd-io/etcd/releases/tag/v3.3.7) (2018-06-06) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.6...v3.3.7) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Security, Authentication - -- Support TLS cipher suite whitelisting. - - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - - TLS handshake fails when client hello is requested with invalid cipher suites. - - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - - If empty, Go auto-populates the list. - -### etcdctl v3 - -- Fix [`etcdctl move-leader` command for TLS-enabled endpoints](https://github.com/etcd-io/etcd/pull/9807). - -### Go - -- Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.6](https://github.com/etcd-io/etcd/releases/tag/v3.3.6) (2018-05-31) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.5...v3.3.6) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### etcd server - -- Allow [empty auth token](https://github.com/etcd-io/etcd/pull/9369). - - Previously, when auth token is an empty string, it returns [`failed to initialize the etcd server: auth: invalid auth options` error](https://github.com/etcd-io/etcd/issues/9349). -- Fix [auth storage panic on server lease revoke routine with JWT token](https://github.com/etcd-io/etcd/issues/9695). -- Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - - Now, this server-side panic has been fixed. - -### Go - -- Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.5](https://github.com/etcd-io/etcd/releases/tag/v3.3.5) (2018-05-09) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.4...v3.3.5) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### etcdctl v3 - -- Fix [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/etcd-io/etcd/pull/9688) parsing. - - Previously, `ETCDCTL_API=3 ./bin/etcdctl watch foo -- echo watch event received` panicked. - -### Go - -- Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.4](https://github.com/etcd-io/etcd/releases/tag/v3.3.4) (2018-04-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.3...v3.3.4) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. -- Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/9557) Prometheus metric. -- Fix [race conditions in v2 server stat collecting](https://github.com/etcd-io/etcd/pull/9562). - -### Security, Authentication - -- Fix [TLS reload](https://github.com/etcd-io/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/etcd-io/etcd/issues/9541). - - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets. - - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/etcd-io/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online. - - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs). - -### etcd server - -- Add [`etcd --initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - - Now, this can be disabled by setting `--initial-election-tick-advance=false`. - - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `etcd --initial-election-tick-advance` at the cost of slow initial bootstrap. - - If single-node, it advances ticks regardless. - - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). - -### Package `embed` - -- Add [`embed.Config.InitialElectionTickAdvance`](https://github.com/etcd-io/etcd/pull/9591) to enable/disable initial election tick fast-forward. - - `embed.NewConfig()` would return `*embed.Config` with `InitialElectionTickAdvance` as true by default. - -### Go - -- Compile with [*Go 1.9.5*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.3](https://github.com/etcd-io/etcd/releases/tag/v3.3.3) (2018-03-29) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.2...v3.3.3) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. -- Adjust [periodic compaction retention window](https://github.com/etcd-io/etcd/pull/9485). - - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). - - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown for every 7.2-hour. **Now, `Compact` happens, for every 1-hour but still with 72-hour retention window.** - - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. **Now, `Compact` happens, for every 30-minute but still with 30-minute retention window.** - - Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`). - - For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data. - - The retention window of compaction period moves for every given compaction period or hour. - - For instance, when hourly writes are 100 and `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour. - - Futhermore, when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute. - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/etcd-io/etcd/pull/9437). - -### Go - -- Compile with [*Go 1.9.5*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.2](https://github.com/etcd-io/etcd/releases/tag/v3.3.2) (2018-03-08) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.1...v3.3.2) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### etcd server - -- Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/etcd-io/etcd/pull/9379). - - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server. - - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`. -- Fix [revision-based compaction retention parsing](https://github.com/etcd-io/etcd/pull/9339). - - Previously, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` was [translated to revision retention 3600000000000](https://github.com/etcd-io/etcd/issues/9337). - - Now, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` is correctly parsed as revision retention 1. -- Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/etcd-io/etcd/pull/9399). - - `TTL` parameter to `Grant` request is unit of second. - - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/etcd-io/etcd/issues/9374). - - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! -- Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/etcd-io/etcd/pull/9347). - -### Proxy v2 - -- Fix [v2 proxy leaky HTTP requests](https://github.com/etcd-io/etcd/pull/9336). - -### Go - -- Compile with [*Go 1.9.4*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.1](https://github.com/etcd-io/etcd/releases/tag/v3.3.1) (2018-02-12) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0...v3.3.1) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Add [warnings on requests taking too long](https://github.com/etcd-io/etcd/pull/9288). - - e.g. `etcdserver: read-only range request "key:\"\\000\" range_end:\"\\000\" " took too long [3.389041388s] to execute` - -### etcd server - -- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9281). - - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. - -### Go - -- Compile with [*Go 1.9.4*](https://golang.org/doc/devel/release.html#go1.9). - - -
- - -## [v3.3.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.0) (2018-02-01) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.0...v3.3.0) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - -- [v3.3.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.0) (2018-02-01), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.4...v3.3.0). -- [v3.3.0-rc.4](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.4) (2018-01-22), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.3...v3.3.0-rc.4). -- [v3.3.0-rc.3](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.3) (2018-01-17), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.2...v3.3.0-rc.3). -- [v3.3.0-rc.2](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.2) (2018-01-11), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.1...v3.3.0-rc.2). -- [v3.3.0-rc.1](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.1) (2018-01-02), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.0...v3.3.0-rc.1). -- [v3.3.0-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.0) (2017-12-20), see [code changes](https://github.com/etcd-io/etcd/compare/v3.2.0...v3.3.0-rc.0). - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** - -### Improved - -- Use [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to replace [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status). - - Fix [etcd database size grows until `mvcc: database space exceeded`](https://github.com/etcd-io/etcd/issues/8009). -- [Support database size larger than 8GiB](https://github.com/etcd-io/etcd/pull/7525) (8GiB is now a suggested maximum size for normal environments) -- [Reduce memory allocation](https://github.com/etcd-io/etcd/pull/8428) on [Range operations](https://github.com/etcd-io/etcd/pull/8475). -- [Rate limit](https://github.com/etcd-io/etcd/pull/8099) and [randomize](https://github.com/etcd-io/etcd/pull/8101) lease revoke on restart or leader elections. - - Prevent [spikes in Raft proposal rate](https://github.com/etcd-io/etcd/issues/8096). -- Support `clientv3` balancer failover under [network faults/partitions](https://github.com/etcd-io/etcd/issues/8711). -- Better warning on [mismatched `etcd --initial-cluster`](https://github.com/etcd-io/etcd/pull/8083) flag. - - etcd compares `etcd --initial-advertise-peer-urls` against corresponding `etcd --initial-cluster` URLs with forward-lookup. - - If resolved IP addresses of `etcd --initial-advertise-peer-urls` and `etcd --initial-cluster` do not match (e.g. [due to DNS error](https://github.com/etcd-io/etcd/pull/9210)), etcd will exit with errors. - - v3.2 error: `etcd --initial-cluster must include s1=https://s1.test:2380 given --initial-advertise-peer-urls=https://s1.test:2380`. - - v3.3 error: `failed to resolve https://s1.test:2380 to match --initial-cluster=s1=https://s1.test:2380 (failed to resolve "https://s1.test:2380" (error ...))`. - -### Breaking Changes - -- Require [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) or [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5). - - Deprecate [`metadata.Incoming/OutgoingContext`](https://github.com/etcd-io/etcd/pull/7896). - - Deprecate `grpclog.Logger`, upgrade to [`grpclog.LoggerV2`](https://github.com/etcd-io/etcd/pull/8533). - - Deprecate [`grpc.ErrClientConnTimeout`](https://github.com/etcd-io/etcd/pull/8505) errors in `clientv3`. - - Use [`MaxRecvMsgSize` and `MaxSendMsgSize`](https://github.com/etcd-io/etcd/pull/8437) to limit message size, in etcd server. -- Translate [gRPC status error in v3 client `Snapshot` API](https://github.com/etcd-io/etcd/pull/9038). -- v3 `etcdctl` [`lease timetolive LEASE_ID`](https://github.com/etcd-io/etcd/issues/9028) on expired lease now prints [`"lease LEASE_ID already expired"`](https://github.com/etcd-io/etcd/pull/9047). - - <=3.2 prints `"lease LEASE_ID granted with TTL(0s), remaining(-1s)"`. -- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3alpha` with [`/v3beta`](https://github.com/etcd-io/etcd/pull/8880). - - To deprecate [`/v3alpha`](https://github.com/etcd-io/etcd/issues/8125) in v3.4. - - In v3.3, `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.4. Use `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. -- Change `etcd --auto-compaction-retention` flag to [accept string values](https://github.com/etcd-io/etcd/pull/8563) with [finer granularity](https://github.com/etcd-io/etcd/issues/8503). - - Now that `etcd --auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type. - - Previously, `--config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`. - - If configured as `etcd --auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `etcd --auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go. - -### Dependency - -- Upgrade [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status) from [**`v1.3.0`**](https://github.com/boltdb/bolt/releases/tag/v1.3.0) to [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) [**`v1.3.1-coreos.6`**](https://github.com/coreos/bbolt/releases/tag/v1.3.1-coreos.6). -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1) to [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5). -- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`v1.1`**](https://github.com/ugorji/go/releases/tag/v1.1), and [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/8721). -- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`ugorji/go@54210f4e0`**](https://github.com/ugorji/go/commit/54210f4e076c57f351166f0ed60e67d3fca57a36), and [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/8574). -- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.2.2`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.2) to [**`v1.3.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0). -- Upgrade [`golang.org/x/crypto/bcrypt`](https://github.com/golang/crypto) to [**`golang/crypto@6c586e17d`**](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117). - -### Metrics, Monitoring - -See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd --listen-metrics-urls`](https://github.com/etcd-io/etcd/pull/8242) flag for additional `/metrics` and `/health` endpoints. - - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/etcd-io/etcd/issues/8060). -- Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). -- Add [`etcd_debugging_mvcc_db_compaction_keys_total`](https://github.com/etcd-io/etcd/pull/8280) Prometheus metric. -- Add [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/8064) Prometheus metric. - - To improve [lease revoke monitoring](https://github.com/etcd-io/etcd/issues/8050). -- Document [Prometheus 2.0 rules](https://github.com/etcd-io/etcd/pull/8879). -- Initialize gRPC server [metrics with zero values](https://github.com/etcd-io/etcd/pull/8878). -- Fix [range/put/delete operation metrics](https://github.com/etcd-io/etcd/pull/8054) with transaction. - - `etcd_debugging_mvcc_range_total` - - `etcd_debugging_mvcc_put_total` - - `etcd_debugging_mvcc_delete_total` - - `etcd_debugging_mvcc_txn_total` -- Fix [`etcd_debugging_mvcc_keys_total`](https://github.com/etcd-io/etcd/pull/8390) on restore. -- Fix [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/8120) on restore. - - Also change to [`prometheus.NewGaugeFunc`](https://github.com/etcd-io/etcd/pull/8150). - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- Add [CRL based connection rejection](https://github.com/etcd-io/etcd/pull/8124) to manage [revoked certs](https://github.com/etcd-io/etcd/issues/4034). -- Document [TLS authentication changes](https://github.com/etcd-io/etcd/pull/8895). - - [Server accepts connections if IP matches, without checking DNS entries](https://github.com/etcd-io/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. - - [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/etcd-io/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. -- Add [`etcd --peer-cert-allowed-cn`](https://github.com/etcd-io/etcd/pull/8616) flag. - - To support [CommonName(CN) based auth](https://github.com/etcd-io/etcd/issues/8262) for inter peer connection. -- [Swap priority](https://github.com/etcd-io/etcd/pull/8594) of cert CommonName(CN) and username + password. - - To address ["username and password specified in the request should take priority over CN in the cert"](https://github.com/etcd-io/etcd/issues/8584). -- Protect [lease revoke with auth](https://github.com/etcd-io/etcd/pull/8031). -- Provide user's role on [auth permission error](https://github.com/etcd-io/etcd/pull/8164). -- Fix [auth store panic with disabled token](https://github.com/etcd-io/etcd/pull/8695). - -### etcd server - -- Add [`etcd --experimental-initial-corrupt-check`](https://github.com/etcd-io/etcd/pull/8554) flag to [check cluster database hashes before serving client/peer traffic](https://github.com/etcd-io/etcd/issues/8313). - - `etcd --experimental-initial-corrupt-check=false` by default. - - v3.4 will enable `--initial-corrupt-check=true` by default. -- Add [`etcd --experimental-corrupt-check-time`](https://github.com/etcd-io/etcd/pull/8420) flag to [raise corrupt alarm monitoring](https://github.com/etcd-io/etcd/issues/7125). - - `etcd --experimental-corrupt-check-time=0s` disabled by default. -- Add [`etcd --experimental-enable-v2v3`](https://github.com/etcd-io/etcd/pull/8407) flag to [emulate v2 API with v3](https://github.com/etcd-io/etcd/issues/6925). - - `etcd --experimental-enable-v2v3=false` by default. -- Add [`etcd --max-txn-ops`](https://github.com/etcd-io/etcd/pull/7976) flag to [configure maximum number operations in transaction](https://github.com/etcd-io/etcd/issues/7826). -- Add [`etcd --max-request-bytes`](https://github.com/etcd-io/etcd/pull/7968) flag to [configure maximum client request size](https://github.com/etcd-io/etcd/issues/7923). - - If not configured, it defaults to 1.5 MiB. -- Add [`etcd --client-crl-file`, `--peer-crl-file`](https://github.com/etcd-io/etcd/pull/8124) flags for [Certificate revocation list](https://github.com/etcd-io/etcd/issues/4034). -- Add [`etcd --peer-cert-allowed-cn`](https://github.com/etcd-io/etcd/pull/8616) flag to support [CN-based auth for inter-peer connection](https://github.com/etcd-io/etcd/issues/8262). -- Add [`etcd --listen-metrics-urls`](https://github.com/etcd-io/etcd/pull/8242) flag for additional `/metrics` and `/health` endpoints. - - Support [additional (non) TLS `/metrics` endpoints for a TLS-enabled cluster](https://github.com/etcd-io/etcd/pull/8282). - - e.g. `etcd --listen-metrics-urls=https://localhost:2378,http://localhost:9379` to serve `/metrics` and `/health` on secure port 2378 and insecure port 9379. - - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/etcd-io/etcd/issues/8060). -- Add [`etcd --auto-compaction-mode`](https://github.com/etcd-io/etcd/pull/8123) flag to [support revision-based compaction](https://github.com/etcd-io/etcd/issues/8098). -- Change `etcd --auto-compaction-retention` flag to [accept string values](https://github.com/etcd-io/etcd/pull/8563) with [finer granularity](https://github.com/etcd-io/etcd/issues/8503). - - Now that `etcd --auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type. - - Previously, `etcd --config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`. - - If configured as `--auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `etcd --auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go. - - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). - - e.g. `etcd --auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown, for every 7.2-hour. - - e.g. `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown, for every 3-minute. - - Periodic compactor continues to record latest revisions for every 1/10 of given compaction period (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=10h`). - - For every 1/10 of given compaction period, compactor uses the last revision that was fetched before compaction period, to discard historical data. - - The retention window of compaction period moves for every 1/10 of given compaction period. - - For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2.x, v3.3.0, v3.3.1, and v3.3.2 compact revision 1000, 1100, and 1200 for every 1-hour. Futhermore, when writes per minute are 1000, v3.3.0, v3.3.1, and v3.3.2 with `--auto-compaction-mode=periodic --auto-compaction-retention=30m` compact revision 30000, 33000, and 36000, for every 3-minute with more finer granularity. - - Whether compaction succeeds or not, this process repeats for every 1/10 of given compaction period. If compaction succeeds, it just removes compacted revision from historical revision records. -- Add [`etcd --grpc-keepalive-min-time`, `etcd --grpc-keepalive-interval`, `etcd --grpc-keepalive-timeout`](https://github.com/etcd-io/etcd/pull/8535) flags to configure server-side keepalive policies. -- Serve [`/health` endpoint as unhealthy](https://github.com/etcd-io/etcd/pull/8272) when [alarm (e.g. `NOSPACE`) is raised or there's no leader](https://github.com/etcd-io/etcd/issues/8207). - - Define [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct with JSON encoder. - - Note that `"health"` field is [`string` type, not `bool`](https://github.com/etcd-io/etcd/pull/9143). - - e.g. `{"health":"false"}`, `{"health":"true"}` - - [Remove `"errors"` field](https://github.com/etcd-io/etcd/pull/9162) since `v3.3.0-rc.3` (did exist only in `v3.3.0-rc.0`, `v3.3.0-rc.1`, `v3.3.0-rc.2`). -- Move [logging setup to embed package](https://github.com/etcd-io/etcd/pull/8810) - - Disable gRPC server info-level logs by default (can be enabled with `etcd --debug` flag). -- Use [monotonic time in Go 1.9](https://github.com/etcd-io/etcd/pull/8507) for `lease` package. -- Warn on [empty hosts in advertise URLs](https://github.com/etcd-io/etcd/pull/8384). - - Address [advertise client URLs accepts empty hosts](https://github.com/etcd-io/etcd/issues/8379). - - etcd v3.4 will exit on this error. - - e.g. `etcd --advertise-client-urls=http://:2379`. -- Warn on [shadowed environment variables](https://github.com/etcd-io/etcd/pull/8385). - - Address [error on shadowed environment variables](https://github.com/etcd-io/etcd/issues/8380). - - etcd v3.4 will exit on this error. - -### API - -- Support [ranges in transaction comparisons](https://github.com/etcd-io/etcd/pull/8025) for [disconnected linearized reads](https://github.com/etcd-io/etcd/issues/7924). -- Add [nested transactions](https://github.com/etcd-io/etcd/pull/8102) to extend [proxy use cases](https://github.com/etcd-io/etcd/issues/7857). -- Add [lease comparison target in transaction](https://github.com/etcd-io/etcd/pull/8324). -- Add [lease list](https://github.com/etcd-io/etcd/pull/8358). -- Add [hash by revision](https://github.com/etcd-io/etcd/pull/8263) for [better corruption checking against boltdb](https://github.com/etcd-io/etcd/issues/8016). - -### client v3 - -- Add [health balancer](https://github.com/etcd-io/etcd/pull/8545) to fix [watch API hangs](https://github.com/etcd-io/etcd/issues/7247), improve [endpoint switch under network faults](https://github.com/etcd-io/etcd/issues/7941). -- [Refactor balancer](https://github.com/etcd-io/etcd/pull/8840) and add [client-side keepalive pings](https://github.com/etcd-io/etcd/pull/8199) to handle [network partitions](https://github.com/etcd-io/etcd/issues/8711). -- Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/etcd-io/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/coreos/etcd/clientv3#Config). - - Fix [exceeded response size limit error in client-side](https://github.com/etcd-io/etcd/issues/9043). - - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099). - - In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB. - - `MaxCallSendMsgSize` default value is 2 MiB, if not configured. - - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured. -- Accept [`Compare_LEASE`](https://github.com/etcd-io/etcd/pull/8324) in [`clientv3.Compare`](https://godoc.org/github.com/coreos/etcd/clientv3#Compare). -- Add [`LeaseValue` helper](https://github.com/etcd-io/etcd/pull/8488) to `Cmp` `LeaseID` values in `Txn`. -- Add [`MoveLeader`](https://github.com/etcd-io/etcd/pull/8153) to `Maintenance`. -- Add [`HashKV`](https://github.com/etcd-io/etcd/pull/8351) to `Maintenance`. -- Add [`Leases`](https://github.com/etcd-io/etcd/pull/8358) to `Lease`. -- Add [`clientv3/ordering`](https://github.com/etcd-io/etcd/pull/8092) for enforce [ordering in serialized requests](https://github.com/etcd-io/etcd/issues/7623). -- Fix ["put at-most-once" violation](https://github.com/etcd-io/etcd/pull/8335). -- Fix [`WatchResponse.Canceled`](https://github.com/etcd-io/etcd/pull/8283) on [compacted watch request](https://github.com/etcd-io/etcd/issues/8231). -- Fix [`concurrency/stm` `Put` with serializable snapshot](https://github.com/etcd-io/etcd/pull/8439). - - Use store revision from first fetch to resolve write conflicts instead of modified revision. - -### etcdctl v3 - -- Add [`etcdctl --discovery-srv`](https://github.com/etcd-io/etcd/pull/8462) flag. -- Add [`etcdctl --keepalive-time`, `--keepalive-timeout`](https://github.com/etcd-io/etcd/pull/8663) flags. -- Add [`etcdctl lease list`](https://github.com/etcd-io/etcd/pull/8358) command. -- Add [`etcdctl lease keep-alive --once`](https://github.com/etcd-io/etcd/pull/8775) flag. -- Make [`lease timetolive LEASE_ID`](https://github.com/etcd-io/etcd/issues/9028) on expired lease print [`lease LEASE_ID already expired`](https://github.com/etcd-io/etcd/pull/9047). - - <=3.2 prints `lease LEASE_ID granted with TTL(0s), remaining(-1s)`. -- Add [`etcdctl snapshot restore --wal-dir`](https://github.com/etcd-io/etcd/pull/9124) flag. -- Add [`etcdctl defrag --data-dir`](https://github.com/etcd-io/etcd/pull/8367) flag. -- Add [`etcdctl move-leader`](https://github.com/etcd-io/etcd/pull/8153) command. -- Add [`etcdctl endpoint hashkv`](https://github.com/etcd-io/etcd/pull/8351) command. -- Add [`etcdctl endpoint --cluster`](https://github.com/etcd-io/etcd/pull/8143) flag, equivalent to [v2 `etcdctl cluster-health`](https://github.com/etcd-io/etcd/issues/8117). -- Make `etcdctl endpoint health` command terminate with [non-zero exit code on unhealthy status](https://github.com/etcd-io/etcd/pull/8342). -- Add [`etcdctl lock --ttl`](https://github.com/etcd-io/etcd/pull/8370) flag. -- Support [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/etcd-io/etcd/pull/8919), equivalent to [v2 `etcdctl exec-watch`](https://github.com/etcd-io/etcd/issues/8814). - - Make `etcdctl watch -- [exec-command]` set environmental variables [`ETCD_WATCH_REVISION`, `ETCD_WATCH_EVENT_TYPE`, `ETCD_WATCH_KEY`, `ETCD_WATCH_VALUE`](https://github.com/etcd-io/etcd/pull/9142) for each event. -- Support [`etcdctl watch` with environmental variables `ETCDCTL_WATCH_KEY` and `ETCDCTL_WATCH_RANGE_END`](https://github.com/etcd-io/etcd/pull/9142). -- Enable [`clientv3.WithRequireLeader(context.Context)` for `watch`](https://github.com/etcd-io/etcd/pull/8672) command. -- Print [`"del"` instead of `"delete"`](https://github.com/etcd-io/etcd/pull/8297) in `txn` interactive mode. -- Print [`ETCD_INITIAL_ADVERTISE_PEER_URLS` in `member add`](https://github.com/etcd-io/etcd/pull/8332). -- Fix [`etcdctl snapshot status` to not modify snapshot file](https://github.com/etcd-io/etcd/pull/8815). - - For example, start etcd `v3.3.10` - - Write some data - - Use etcdctl `v3.3.10` to save snapshot - - Somehow, upgrading Kubernetes fails, thus rolling back to previous version etcd `v3.2.24` - - Run etcdctl `v3.2.24` `snapshot status` against the snapshot file saved from `v3.3.10` server - - Run etcdctl `v3.2.24` `snapshot restore` fails with `"expected sha256 [12..."` - -### etcdctl v3 - -- Handle [empty key permission](https://github.com/etcd-io/etcd/pull/8514) in `etcdctl`. - -### etcdctl v2 - -- Add [`etcdctl backup --with-v3`](https://github.com/etcd-io/etcd/pull/8479) flag. - -### gRPC Proxy - -- Add [`grpc-proxy start --experimental-leasing-prefix`](https://github.com/etcd-io/etcd/pull/8341) flag. - - For disconnected linearized reads. - - Based on [V system leasing](https://github.com/etcd-io/etcd/issues/6065). - - See ["Disconnected consistent reads with etcd" blog post](https://coreos.com/blog/coreos-labs-disconnected-consistent-reads-with-etcd). -- Add [`grpc-proxy start --experimental-serializable-ordering`](https://github.com/etcd-io/etcd/pull/8315) flag. - - To ensure serializable reads have monotonically increasing store revisions across endpoints. -- Add [`grpc-proxy start --metrics-addr`](https://github.com/etcd-io/etcd/pull/8242) flag for an additional `/metrics` endpoint. - - Set `--metrics-addr=http://[HOST]:9379` to serve `/metrics` in insecure port 9379. -- Serve [`/health` endpoint in grpc-proxy](https://github.com/etcd-io/etcd/pull/8322). -- Add [`grpc-proxy start --debug`](https://github.com/etcd-io/etcd/pull/8994) flag. -- Add [`grpc-proxy start --max-send-bytes`](https://github.com/etcd-io/etcd/pull/9250) flag to [configure maximum client request size](https://github.com/etcd-io/etcd/issues/7923). -- Add [`grpc-proxy start --max-recv-bytes`](https://github.com/etcd-io/etcd/pull/9250) flag to [configure maximum client request size](https://github.com/etcd-io/etcd/issues/7923). -- Fix [Snapshot API error handling](https://github.com/etcd-io/etcd/commit/dbd16d52fbf81e5fd806d21ff5e9148d5bf203ab). -- Fix [KV API `PrevKv` flag handling](https://github.com/etcd-io/etcd/pull/8366). -- Fix [KV API `KeysOnly` flag handling](https://github.com/etcd-io/etcd/pull/8552). - -### gRPC gateway - -- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3alpha` with [`/v3beta`](https://github.com/etcd-io/etcd/pull/8880). - - To deprecate [`/v3alpha`](https://github.com/etcd-io/etcd/issues/8125) in v3.4. - - In v3.3, `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.4. Use `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. -- Support ["authorization" token](https://github.com/etcd-io/etcd/pull/7999). -- Support [websocket for bi-directional streams](https://github.com/etcd-io/etcd/pull/8257). - - Fix [`Watch` API with gRPC gateway](https://github.com/etcd-io/etcd/issues/8237). -- Upgrade gRPC gateway to [v1.3.0](https://github.com/etcd-io/etcd/issues/8838). - -### etcd server - -- Fix [backend database in-memory index corruption](https://github.com/etcd-io/etcd/pull/8127) issue on restore (only 3.2.0 is affected). -- Fix [watch restore from snapshot](https://github.com/etcd-io/etcd/pull/8427). -- Fix [`mvcc/backend.defragdb` nil-pointer dereference on create bucket failure](https://github.com/etcd-io/etcd/pull/9119). -- Fix [server crash](https://github.com/etcd-io/etcd/pull/8010) on [invalid transaction request from gRPC gateway](https://github.com/etcd-io/etcd/issues/7889). -- Prevent [server panic from member update/add](https://github.com/etcd-io/etcd/pull/9174) with [wrong scheme URLs](https://github.com/etcd-io/etcd/issues/9173). -- Make [peer dial timeout longer](https://github.com/etcd-io/etcd/pull/8599). - - See [coreos/etcd-operator#1300](https://github.com/etcd-io/etcd-operator/issues/1300) for more detail. -- Make server [wait up to request time-out](https://github.com/etcd-io/etcd/pull/8267) with [pending RPCs](https://github.com/etcd-io/etcd/issues/8224). -- Fix [`grpc.Server` panic on `GracefulStop`](https://github.com/etcd-io/etcd/pull/8987) with [TLS-enabled server](https://github.com/etcd-io/etcd/issues/8916). -- Fix ["multiple peer URLs cannot start" issue](https://github.com/etcd-io/etcd/issues/8383). -- Fix server-side auth so [concurrent auth operations do not return old revision error](https://github.com/etcd-io/etcd/pull/8442). -- Handle [WAL renaming failure on Windows](https://github.com/etcd-io/etcd/pull/8286). -- Upgrade [`coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) to `v15` (see https://github.com/coreos/go-systemd/releases/tag/v15). -- [Put back `/v2/machines`](https://github.com/etcd-io/etcd/pull/8062) endpoint for python-etcd wrapper. - -### client v2 - -- [Fail-over v2 client](https://github.com/etcd-io/etcd/pull/8519) to next endpoint on [oneshot failure](https://github.com/etcd-io/etcd/issues/8515). - -### Package `raft` - -- Add [non-voting member](https://github.com/etcd-io/etcd/pull/8751). - - To implement [Raft thesis 4.2.1 Catching up new servers](https://github.com/etcd-io/etcd/issues/8568). - - `Learner` node does not vote or promote itself. - -### Other - -- Support previous two minor versions (see our [new release policy](https://github.com/etcd-io/etcd/pull/8805)). -- `v3.3.x` is the last release cycle that supports `ACI`. - - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016. - - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore. - - `*.aci` files won't be available from etcd v3.4 release. -- Add container registry [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd). - - [quay.io/coreos/etcd](https://quay.io/coreos/etcd) is still supported as secondary. - -### Go - -- Require [*Go 1.9+*](https://github.com/etcd-io/etcd/issues/6174). -- Compile with [*Go 1.9.3*](https://golang.org/doc/devel/release.html#go1.9). -- Deprecate [`golang.org/x/net/context`](https://github.com/etcd-io/etcd/pull/8511). - - -
- diff --git a/CHANGELOG-3.4.md b/CHANGELOG-3.4.md deleted file mode 100644 index 63926da1eead..000000000000 --- a/CHANGELOG-3.4.md +++ /dev/null @@ -1,1040 +0,0 @@ - - -Previous change logs can be found at [CHANGELOG-3.3](https://github.com/etcd-io/etcd/blob/main/CHANGELOG-3.3.md). - - -The minimum recommended etcd versions to run in **production** are 3.2.28+, 3.3.18+, and 3.4.2+. - -
- -## v3.4.16 (2021-05-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.15...v3.4.16) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### etcd server - -- Add [`--experimental-warning-apply-duration`](https://github.com/etcd-io/etcd/pull/12448) flag which allows apply duration threshold to be configurable. -- Fix [`--unsafe-no-fsync`](https://github.com/etcd-io/etcd/pull/12751) to still write-out data avoiding corruption (most of the time). -- Reduce [around 30% memory allocation by logging range response size without marshal](https://github.com/etcd-io/etcd/pull/12871). -- Add [exclude alarms from health check conditionally](https://github.com/etcd-io/etcd/pull/12880). - -### Metrics - -- Fix [incorrect metrics generated when clients cancel watches](https://github.com/etcd-io/etcd/pull/12803) back-ported from (https://github.com/etcd-io/etcd/pull/12196). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.15](https://github.com/etcd-io/etcd/releases/tag/v3.4.15) (2021-02-26) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.14...v3.4.15) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### etcd server - -- Log [successful etcd server-side health check in debug level](https://github.com/etcd-io/etcd/pull/12677). -- Fix [64 KB websocket notification message limit](https://github.com/etcd-io/etcd/pull/12402). - -### Package `fileutil` - -- Fix [`F_OFD_` constants](https://github.com/etcd-io/etcd/pull/12444). - -### Dependency - -- Bump up [`gorilla/websocket` to v1.4.2](https://github.com/etcd-io/etcd/pull/12645). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.14](https://github.com/etcd-io/etcd/releases/tag/v3.4.14) (2020-11-25) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.13...v3.4.14) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### Package `clientv3` - -- Fix [auth token invalid after watch reconnects](https://github.com/etcd-io/etcd/pull/12264). Get AuthToken automatically when clientConn is ready. - -### etcd server - -- [Fix server panic](https://github.com/etcd-io/etcd/pull/12288) when force-new-cluster flag is enabled in a cluster which had learner node. - -### Package `netutil` - -- Remove [`netutil.DropPort/RecoverPort/SetLatency/RemoveLatency`](https://github.com/etcd-io/etcd/pull/12491). - - These are not used anymore. They were only used for older versions of functional testing. - - Removed to adhere to best security practices, minimize arbitrary shell invocation. - -### `tools/etcd-dump-metrics` - -- Implement [input validation to prevent arbitrary shell invocation](https://github.com/etcd-io/etcd/pull/12491). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.13](https://github.com/etcd-io/etcd/releases/tag/v3.4.13) (2020-8-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.12...v3.4.13) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### Security - -- A [log warning](https://github.com/etcd-io/etcd/pull/12242) is added when etcd use any existing directory that has a permission different than 700 on Linux and 777 on Windows. - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.12](https://github.com/etcd-io/etcd/releases/tag/v3.4.12) (2020-08-19) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.11...v3.4.12) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### etcd server - -- Fix [server panic in slow writes warnings](https://github.com/etcd-io/etcd/issues/12197). - - Fixed via [PR#12238](https://github.com/etcd-io/etcd/pull/12238). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - - -
- - - -## [v3.4.11](https://github.com/etcd-io/etcd/releases/tag/v3.4.11) (2020-08-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.10...v3.4.11) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### etcd server - -- Improve [`runtime.FDUsage` call pattern to reduce objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). -- Add [`etcd --experimental-watch-progress-notify-interval`](https://github.com/etcd-io/etcd/pull/12216) flag to make watch progress notify interval configurable. - -### Package `clientv3` - -- Remove [excessive watch cancel logging messages](https://github.com/etcd-io/etcd/pull/12187). - - See [kubernetes/kubernetes#93450](https://github.com/kubernetes/kubernetes/issues/93450). - -### Package `runtime` - -- Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). - -### Metrics, Monitoring - -- Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - - - -
- - - - -## [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) (2020-07-16) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.9...v3.4.10) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### Package `etcd server` - -- Add [`--unsafe-no-fsync`](https://github.com/etcd-io/etcd/pull/11946) flag. - - Setting the flag disables all uses of fsync, which is unsafe and will cause data loss. This flag makes it possible to run an etcd node for testing and development without placing lots of load on the file system. -- Add [etcd --auth-token-ttl](https://github.com/etcd-io/etcd/pull/11980) flag to customize `simpleTokenTTL` settings. -- Improve [runtime.FDUsage objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). -- Improve [mvcc.watchResponse channel Memory Usage](https://github.com/etcd-io/etcd/pull/11987). -- Fix [`int64` convert panic in raft logger](https://github.com/etcd-io/etcd/pull/12106). - - Fix [kubernetes/kubernetes#91937](https://github.com/kubernetes/kubernetes/issues/91937). - -### Breaking Changes - -- Changed behavior on [existing dir permission](https://github.com/etcd-io/etcd/pull/11798). - - Previously, the permission was not checked on existing data directory and the directory used for automatically generating self-signed certificates for TLS connections with clients. Now a check is added to make sure those directories, if already exist, has a desired permission of 700 on Linux and 777 on Windows. - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.9](https://github.com/etcd-io/etcd/releases/tag/v3.4.9) (2020-05-20) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.8...v3.4.9) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### Package `wal` - -- Add [missing CRC checksum check in WAL validate method otherwise causes panic](https://github.com/etcd-io/etcd/pull/11924). - - See https://github.com/etcd-io/etcd/issues/11918. - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.8](https://github.com/etcd-io/etcd/releases/tag/v3.4.8) (2020-05-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.7...v3.4.8) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### `etcdctl` - -- Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). - -### Package `clientv3` - -- Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). - -### etcd server - -- Improve logging around snapshot send and receive. -- [Add log when etcdserver failed to apply command](https://github.com/etcd-io/etcd/pull/11670). -- [Fix deadlock bug in mvcc](https://github.com/etcd-io/etcd/pull/11817). -- Fix [inconsistency between WAL and server snapshot](https://github.com/etcd-io/etcd/pull/11888). - - Previously, server restore fails if it had crashed after persisting raft hard state but before saving snapshot. - - See https://github.com/etcd-io/etcd/issues/10219 for more. - -### Package Auth - -- [Fix a data corruption bug by saving consistent index](https://github.com/etcd-io/etcd/pull/11652). - -### Metrics, Monitoring - -- Add [`etcd_debugging_auth_revision`](https://github.com/etcd-io/etcd/commit/f14d2a087f7b0fd6f7980b95b5e0b945109c95f3). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.7](https://github.com/etcd-io/etcd/releases/tag/v3.4.7) (2020-04-01) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.6...v3.4.7) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### etcd server - -- Improve [compaction performance when latest index is greater than 1-million](https://github.com/etcd-io/etcd/pull/11734). - -### Package `wal` - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - -### Metrics, Monitoring - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.6](https://github.com/etcd-io/etcd/releases/tag/v3.4.6) (2020-03-29) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.5...v3.4.6) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -### Package `lease` - -- Fix [memory leak in follower nodes](https://github.com/etcd-io/etcd/pull/11731). - - https://github.com/etcd-io/etcd/issues/11495 - - https://github.com/etcd-io/etcd/issues/11730 - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) (2020-03-18) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.4...v3.4.5) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** - -### etcd server - -- Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). - -### client v3 - -- Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. - -### etcdctl v3 - -- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11638) command to prevent potential timeout. - -### Metrics, Monitoring - -See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - -- Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). - -### gRPC Proxy - -- Fix [`panic on error`](https://github.com/etcd-io/etcd/pull/11694) for metrics handler. - -### Go - -- Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) (2020-02-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.3...v3.4.4) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** - -### etcd server - -- Fix [`wait purge file loop during shutdown`](https://github.com/etcd-io/etcd/pull/11308). - - Previously, during shutdown etcd could accidentally remove needed wal files, resulting in catastrophic error `etcdserver: open wal error: wal: file not found.` during startup. - - Now, etcd makes sure the purge file loop exits before server signals stop of the raft node. -- [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). -- Fix [quorum protection logic when promoting a learner](https://github.com/etcd-io/etcd/pull/11640). -- Improve [peer corruption checker](https://github.com/etcd-io/etcd/pull/11621) to work when peer mTLS is enabled. - -### Metrics, Monitoring - -See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_debugging_mvcc_total_put_size_in_bytes`](https://github.com/etcd-io/etcd/pull/11374) Prometheus metric. -- Fix bug where [etcd_debugging_mvcc_db_compaction_keys_total is always 0](https://github.com/etcd-io/etcd/pull/11400). - -### Auth - -- Fix [NoPassword check when adding user through GRPC gateway](https://github.com/etcd-io/etcd/pull/11418) ([issue#11414](https://github.com/etcd-io/etcd/issues/11414)) -- Fix bug where [some auth related messages are logged at wrong level](https://github.com/etcd-io/etcd/pull/11586) - - -
- - -## [v3.4.3](https://github.com/etcd-io/etcd/releases/tag/v3.4.3) (2019-10-24) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.2...v3.4.3) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** - -### Metrics, Monitoring - -See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Change [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11254) Prometheus metrics to include only major and minor version. - -### Go - -- Compile with [*Go 1.12.12*](https://golang.org/doc/devel/release.html#go1.12). - - -
- - -## [v3.4.2](https://github.com/etcd-io/etcd/releases/tag/v3.4.2) (2019-10-11) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.1...v3.4.2) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** - -### etcdctl v3 - -- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout. - -### etcdserver - -- Add [`tracing`](https://github.com/etcd-io/etcd/pull/11179) to range, put and compact requests in etcdserver. - -### Go - -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - -### client v3 - -- Fix [client balancer failover against multiple endpoints](https://github.com/etcd-io/etcd/pull/11184). - - Fix ["kube-apiserver: failover on multi-member etcd cluster fails certificate check on DNS mismatch" (kubernetes#83028)](https://github.com/kubernetes/kubernetes/issues/83028). -- Fix [IPv6 endpoint parsing in client](https://github.com/etcd-io/etcd/pull/11211). - - Fix ["1.16: etcd client does not parse IPv6 addresses correctly when members are joining" (kubernetes#83550)](https://github.com/kubernetes/kubernetes/issues/83550). - - -
- - -## [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) (2019-09-17) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.4.1) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** - -### Metrics, Monitoring - -See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. -- Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - -### etcd server - -- Fix [secure server logging message](https://github.com/etcd-io/etcd/commit/8b053b0f44c14ac0d9f39b9b78c17c57d47966eb). -- Remove [redundant `%` characters in file descriptor warning message](https://github.com/etcd-io/etcd/commit/d5f79adc9cea9ec8c93669526464b0aa19ed417b). - -### Package `embed` - -- Add [`embed.Config.ZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/11148) to allow creating a custom zap logger. - -### Dependency - -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0) to [**`v1.23.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.1). - -### Go - -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - - -
- - -## v3.4.0 (2019-08-30) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0...v3.4.0) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - -- [v3.4.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0) (2019-08-30), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.4...v3.4.0). -- [v3.4.0-rc.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.4) (2019-08-29), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.3...v3.4.0-rc.4). -- [v3.4.0-rc.3](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.3) (2019-08-27), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.2...v3.4.0-rc.3). -- [v3.4.0-rc.2](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.2) (2019-08-23), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.1...v3.4.0-rc.2). -- [v3.4.0-rc.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.1) (2019-08-15), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.0...v3.4.0-rc.1). -- [v3.4.0-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.0) (2019-08-12), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0...v3.4.0-rc.0). - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** - -### Documentation - -- etcd now has a new website! Please visit https://etcd.io. - -### Improved - -- Add Raft learner: [etcd#10725](https://github.com/etcd-io/etcd/pull/10725), [etcd#10727](https://github.com/etcd-io/etcd/pull/10727), [etcd#10730](https://github.com/etcd-io/etcd/pull/10730). - - User guide: [runtime-configuration document](https://etcd.io/docs/latest/op-guide/runtime-configuration/#add-a-new-member-as-learner). - - API change: [API reference document](https://etcd.io/docs/latest/dev-guide/api_reference_v3/). - - More details on implementation: [learner design document](https://etcd.io/docs/latest/learning/design-learner/) and [implementation task list](https://github.com/etcd-io/etcd/issues/10537). -- Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - - Fix [gRPC panic "send on closed channel](https://github.com/etcd-io/etcd/issues/9956). - - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. To block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. -- Add [backoff on watch retries on transient errors](https://github.com/etcd-io/etcd/pull/9840). -- Add [jitter to watch progress notify](https://github.com/etcd-io/etcd/pull/9278) to prevent [spikes in `etcd_network_client_grpc_sent_bytes_total`](https://github.com/etcd-io/etcd/issues/9246). -- Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. -- Improve [slow request apply warning log](https://github.com/etcd-io/etcd/pull/9288). - - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`. - - Redact [request value field](https://github.com/etcd-io/etcd/pull/9822). - - Provide [response size](https://github.com/etcd-io/etcd/pull/9826). -- Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. -- Improve [TLS setup error logging](https://github.com/etcd-io/etcd/pull/9518) to help debug [TLS-enabled cluster configuring issues](https://github.com/etcd-io/etcd/issues/9400). -- Improve [long-running concurrent read transactions under light write workloads](https://github.com/etcd-io/etcd/pull/9296). - - Previously, periodic commit on pending writes blocks incoming read transactions, even if there is no pending write. - - Now, periodic commit operation does not block concurrent read transactions, thus improves long-running read transaction performance. -- Make [backend read transactions fully concurrent](https://github.com/etcd-io/etcd/pull/10523). - - Previously, ongoing long-running read transactions block writes and future reads. - - With this change, write throughput is increased by 70% and P99 write latency is reduced by 90% in the presence of long-running reads. -- Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). -- Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. -- Add [Raft Pre-Vote feature](https://github.com/etcd-io/etcd/pull/9352) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - - For instance, a flaky(or rejoining) member may drop in and out, and start campaign. This member will end up with a higher term, and ignore all incoming messages with lower term. In this case, a new leader eventually need to get elected, thus disruptive to cluster availability. Raft implements Pre-Vote phase to prevent this kind of disruptions. If enabled, Raft runs an additional phase of election to check if pre-candidate can get enough votes to win an election. -- Adjust [periodic compaction retention window](https://github.com/etcd-io/etcd/pull/9485). - - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). - - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h` automatically `Compact` with 24-hour retention windown for every 2.4-hour. Now, `Compact` happens for every 1-hour. - - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. Now, `Compact` happens for every 30-minute. - - Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`). - - For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data. - - The retention window of compaction period moves for every given compaction period or hour. - - For instance, when hourly writes are 100 and `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour. - - Futhermore, when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute. -- Improve [lease expire/revoke operation performance](https://github.com/etcd-io/etcd/pull/9418), address [lease scalability issue](https://github.com/etcd-io/etcd/issues/9496). -- Make [Lease `Lookup` non-blocking with concurrent `Grant`/`Revoke`](https://github.com/etcd-io/etcd/pull/9229). -- Make etcd server return `raft.ErrProposalDropped` on internal Raft proposal drop in [v3 applier](https://github.com/etcd-io/etcd/pull/9549) and [v2 applier](https://github.com/etcd-io/etcd/pull/9558). - - e.g. a node is removed from cluster, or [`raftpb.MsgProp` arrives at current leader while there is an ongoing leadership transfer](https://github.com/etcd-io/etcd/issues/8975). -- Add [`snapshot`](https://github.com/etcd-io/etcd/pull/9118) package for easier snapshot workflow (see [`godoc.org/github.com/etcd/clientv3/snapshot`](https://godoc.org/github.com/etcd-io/etcd/clientv3/snapshot) for more). -- Improve [functional tester](https://github.com/etcd-io/etcd/tree/main/functional) coverage: [proxy layer to run network fault tests in CI](https://github.com/etcd-io/etcd/pull/9081), [TLS is enabled both for server and client](https://github.com/etcd-io/etcd/pull/9534), [liveness mode](https://github.com/etcd-io/etcd/issues/9230), [shuffle test sequence](https://github.com/etcd-io/etcd/issues/9381), [membership reconfiguration failure cases](https://github.com/etcd-io/etcd/pull/9564), [disastrous quorum loss and snapshot recover from a seed member](https://github.com/etcd-io/etcd/pull/9565), [embedded etcd](https://github.com/etcd-io/etcd/pull/9572). -- Improve [index compaction blocking](https://github.com/etcd-io/etcd/pull/9511) by using a copy on write clone to avoid holding the lock for the traversal of the entire index. -- Update [JWT methods](https://github.com/etcd-io/etcd/pull/9883) to allow for use of any supported signature method/algorithm. -- Add [Lease checkpointing](https://github.com/etcd-io/etcd/pull/9924) to persist remaining TTLs to the consensus log periodically so that long lived leases progress toward expiry in the presence of leader elections and server restarts. - - Enabled by experimental flag "--experimental-enable-lease-checkpoint". -- Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. -- Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. -- Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). -- Improve [heartbeat send failure logging](https://github.com/etcd-io/etcd/pull/10663). -- Support [users with no password](https://github.com/etcd-io/etcd/pull/9817) for reducing security risk introduced by leaked password. The users can only be authenticated with `CommonName` based auth. -- Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/10524). -- Add `etcd --experimental-compaction-batch-limit` to [sets the maximum revisions deleted in each compaction batch](https://github.com/etcd-io/etcd/pull/11034). -- Reduced default compaction batch size from 10k revisions to 1k revisions to improve p99 latency during compactions and reduced wait between compactions from 100ms to 10ms. - -### Breaking Changes - -- Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - - Fix [gRPC panic "send on closed channel](https://github.com/etcd-io/etcd/issues/9956). - - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. To block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. -- Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). - - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. -- Migrate dependency management tool from `glide` to [Go module](https://github.com/etcd-io/etcd/pull/10063). - - <= 3.3 puts `vendor` directory under `cmd/vendor` directory to [prevent conflicting transitive dependencies](https://github.com/etcd-io/etcd/issues/4913). - - 3.4 moves `cmd/vendor` directory to `vendor` at repository root. - - Remove recursive symlinks in `cmd` directory. - - Now `go get/install/build` on `etcd` packages (e.g. `clientv3`, `tools/benchmark`) enforce builds with etcd `vendor` directory. -- Deprecated `latest` [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tag. - - **`docker pull gcr.io/etcd-development/etcd:latest` would not be up-to-date**. -- Deprecated [minor](https://semver.org/) version [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tags. - - `docker pull gcr.io/etcd-development/etcd:v3.3` would still work. - - **`docker pull gcr.io/etcd-development/etcd:v3.4` would not work**. - - Use **`docker pull gcr.io/etcd-development/etcd:v3.4.x`** instead, with the exact patch version. -- Deprecated [ACIs from official release](https://github.com/etcd-io/etcd/pull/9059). - - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016. - - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore. - - `*.aci` files are not available from `v3.4` release. -- Move [`"github.com/coreos/etcd"`](https://github.com/etcd-io/etcd/issues/9965) to [`"github.com/etcd-io/etcd"`](https://github.com/etcd-io/etcd/issues/9965). - - Change import path to `"go.etcd.io/etcd"`. - - e.g. `import "go.etcd.io/etcd/raft"`. -- Make [`ETCDCTL_API=3 etcdctl` default](https://github.com/etcd-io/etcd/issues/9600). - - Now, `etcdctl set foo bar` must be `ETCDCTL_API=2 etcdctl set foo bar`. - - Now, `ETCDCTL_API=3 etcdctl put foo bar` could be just `etcdctl put foo bar`. -- Make [`etcd --enable-v2=false` default](https://github.com/etcd-io/etcd/pull/10935). -- Make [`embed.DefaultEnableV2` `false` default](https://github.com/etcd-io/etcd/pull/10935). -- **Deprecated `etcd --ca-file` flag**. Use [`etcd --trusted-ca-file`](https://github.com/etcd-io/etcd/pull/9470) instead (`etcd --ca-file` flag has been marked deprecated since v2.1). -- **Deprecated `etcd --peer-ca-file` flag**. Use [`etcd --peer-trusted-ca-file`](https://github.com/etcd-io/etcd/pull/9470) instead (`etcd --peer-ca-file` flag has been marked deprecated since v2.1). -- **Deprecated `pkg/transport.TLSInfo.CAFile` field**. Use [`pkg/transport.TLSInfo.TrustedCAFile`](https://github.com/etcd-io/etcd/pull/9470) instead (`CAFile` field has been marked deprecated since v2.1). -- Exit on [empty hosts in advertise URLs](https://github.com/etcd-io/etcd/pull/8786). - - Address [advertise client URLs accepts empty hosts](https://github.com/etcd-io/etcd/issues/8379). - - e.g. exit with error on `--advertise-client-urls=http://:2379`. - - e.g. exit with error on `--initial-advertise-peer-urls=http://:2380`. -- Exit on [shadowed environment variables](https://github.com/etcd-io/etcd/pull/9382). - - Address [error on shadowed environment variables](https://github.com/etcd-io/etcd/issues/8380). - - e.g. exit with error on `ETCD_NAME=abc etcd --name=def`. - - e.g. exit with error on `ETCD_INITIAL_CLUSTER_TOKEN=abc etcd --initial-cluster-token=def`. - - e.g. exit with error on `ETCDCTL_ENDPOINTS=abc.com ETCDCTL_API=3 etcdctl endpoint health --endpoints=def.com`. -- Change [`etcdserverpb.AuthRoleRevokePermissionRequest/key,range_end` fields type from `string` to `bytes`](https://github.com/etcd-io/etcd/pull/9433). -- Deprecating `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) instead. -- Deprecating `etcd_debugging_mvcc_put_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_put_total`](https://github.com/etcd-io/etcd/pull/10962) instead. -- Deprecating `etcd_debugging_mvcc_delete_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_delete_total`](https://github.com/etcd-io/etcd/pull/10962) instead. -- Deprecating `etcd_debugging_mvcc_range_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_range_total`](https://github.com/etcd-io/etcd/pull/10968) instead. -- Deprecating `etcd_debugging_mvcc_txn_total`Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_txn_total`](https://github.com/etcd-io/etcd/pull/10968) instead. -- Rename `etcdserver.ServerConfig.SnapCount` field to `etcdserver.ServerConfig.SnapshotCount`, to be consistent with the flag name `etcd --snapshot-count`. -- Rename `embed.Config.SnapCount` field to [`embed.Config.SnapshotCount`](https://github.com/etcd-io/etcd/pull/9745), to be consistent with the flag name `etcd --snapshot-count`. -- Change [`embed.Config.CorsInfo` in `*cors.CORSInfo` type to `embed.Config.CORS` in `map[string]struct{}` type](https://github.com/etcd-io/etcd/pull/9490). -- Deprecated [`embed.Config.SetupLogging`](https://github.com/etcd-io/etcd/pull/9572). - - Now logger is set up automatically based on [`embed.Config.Logger`, `embed.Config.LogOutputs`, `embed.Config.Debug` fields](https://github.com/etcd-io/etcd/pull/9572). -- Rename [`etcd --log-output` to `etcd --log-outputs`](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. - - **`etcd --log-output`** will be deprecated in v3.5. -- Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. -- Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/etcd-io/etcd/pull/9579) to support multiple log outputs. - - Now that `etcd --log-outputs` accepts multiple writers, etcd configuration YAML file `log-outputs` field must be changed to `[]string` type. - - Previously, `etcd --config-file etcd.config.yaml` can have `log-outputs: default` field, now must be `log-outputs: [default]`. -- Deprecating [`etcd --debug`](https://github.com/etcd-io/etcd/pull/10947) flag. Use `etcd --log-level=debug` flag instead. - - v3.5 will deprecate `etcd --debug` flag in favor of `etcd --log-level=debug`. -- Change v3 `etcdctl snapshot` exit codes with [`snapshot` package](https://github.com/etcd-io/etcd/pull/9118/commits/df689f4280e1cce4b9d61300be13ca604d41670a). - - Exit on error with exit code 1 (no more exit code 5 or 6 on `snapshot save/restore` commands). -- Deprecated [`grpc.ErrClientConnClosing`](https://github.com/etcd-io/etcd/pull/10981). - - `clientv3` and `proxy/grpcproxy` now does not return `grpc.ErrClientConnClosing`. - - `grpc.ErrClientConnClosing` has been [deprecated in gRPC >= 1.10](https://github.com/grpc/grpc-go/pull/1854). - - Use `clientv3.IsConnCanceled(error)` or `google.golang.org/grpc/status.FromError(error)` instead. -- Deprecated [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3beta` with [`/v3`](https://github.com/etcd-io/etcd/pull/9298). - - Deprecated [`/v3alpha`](https://github.com/etcd-io/etcd/pull/9298). - - To deprecate [`/v3beta`](https://github.com/etcd-io/etcd/issues/9189) in v3.5. - - In v3.4, `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. -- Change [`wal` package function signatures](https://github.com/etcd-io/etcd/pull/9572) to support [structured logger and logging to file](https://github.com/etcd-io/etcd/issues/9438) in server-side. - - Previously, `Open(dirpath string, snap walpb.Snapshot) (*WAL, error)`, now `Open(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error)`. - - Previously, `OpenForRead(dirpath string, snap walpb.Snapshot) (*WAL, error)`, now `OpenForRead(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error)`. - - Previously, `Repair(dirpath string) bool`, now `Repair(lg *zap.Logger, dirpath string) bool`. - - Previously, `Create(dirpath string, metadata []byte) (*WAL, error)`, now `Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error)`. -- Remove [`pkg/cors` package](https://github.com/etcd-io/etcd/pull/9490). -- Move internal packages to `etcdserver`. - - `"github.com/coreos/etcd/alarm"` to `"go.etcd.io/etcd/etcdserver/api/v3alarm"`. - - `"github.com/coreos/etcd/compactor"` to `"go.etcd.io/etcd/etcdserver/api/v3compactor"`. - - `"github.com/coreos/etcd/discovery"` to `"go.etcd.io/etcd/etcdserver/api/v2discovery"`. - - `"github.com/coreos/etcd/etcdserver/auth"` to `"go.etcd.io/etcd/etcdserver/api/v2auth"`. - - `"github.com/coreos/etcd/etcdserver/membership"` to `"go.etcd.io/etcd/etcdserver/api/membership"`. - - `"github.com/coreos/etcd/etcdserver/stats"` to `"go.etcd.io/etcd/etcdserver/api/v2stats"`. - - `"github.com/coreos/etcd/error"` to `"go.etcd.io/etcd/etcdserver/api/v2error"`. - - `"github.com/coreos/etcd/rafthttp"` to `"go.etcd.io/etcd/etcdserver/api/rafthttp"`. - - `"github.com/coreos/etcd/snap"` to `"go.etcd.io/etcd/etcdserver/api/snap"`. - - `"github.com/coreos/etcd/store"` to `"go.etcd.io/etcd/etcdserver/api/v2store"`. -- Change [snapshot file permissions](https://github.com/etcd-io/etcd/pull/9977): On Linux, the snapshot file changes from readable by all (mode 0644) to readable by the user only (mode 0600). -- Change [`pkg/adt.IntervalTree` from `struct` to `interface`](https://github.com/etcd-io/etcd/pull/10959). - - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). -- Release branch `/version` defines version `3.4.x-pre`, instead of `3.4.y+git`. - - Use `3.4.5-pre`, instead of `3.4.4+git`. - -### Dependency - -- Upgrade [`github.com/coreos/bbolt`](https://github.com/etcd-io/bbolt/releases) from [**`v1.3.1-coreos.6`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.1-coreos.6) to [`go.etcd.io/bbolt`](https://github.com/etcd-io/bbolt/releases) [**`v1.3.3`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3). -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5) to [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0). -- Migrate [`github.com/ugorji/go/codec`](https://github.com/ugorji/go/releases) to [**`github.com/json-iterator/go`**](https://github.com/json-iterator/go), to [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/9494) (See [#10667](https://github.com/etcd-io/etcd/pull/10667) for more). -- Migrate [`github.com/ghodss/yaml`](https://github.com/ghodss/yaml/releases) to [**`sigs.k8s.io/yaml`**](https://github.com/kubernetes-sigs/yaml) (See [#10687](https://github.com/etcd-io/etcd/pull/10687) for more). -- Upgrade [`golang.org/x/crypto`](https://github.com/golang/crypto) from [**`crypto@9419663f5`**](https://github.com/golang/crypto/commit/9419663f5a44be8b34ca85f08abc5fe1be11f8a3) to [**`crypto@0709b304e793`**](https://github.com/golang/crypto/commit/0709b304e793a5edb4a2c0145f281ecdc20838a4). -- Upgrade [`golang.org/x/net`](https://github.com/golang/net) from [**`net@66aacef3d`**](https://github.com/golang/net/commit/66aacef3dd8a676686c7ae3716979581e8b03c47) to [**`net@adae6a3d119a`**](https://github.com/golang/net/commit/adae6a3d119ae4890b46832a2e88a95adc62b8e7). -- Upgrade [`golang.org/x/sys`](https://github.com/golang/sys) from [**`sys@ebfc5b463`**](https://github.com/golang/sys/commit/ebfc5b4631820b793c9010c87fd8fef0f39eb082) to [**`sys@c7b8b68b1456`**](https://github.com/golang/sys/commit/c7b8b68b14567162c6602a7c5659ee0f26417c18). -- Upgrade [`golang.org/x/text`](https://github.com/golang/text) from [**`text@b19bf474d`**](https://github.com/golang/text/commit/b19bf474d317b857955b12035d2c5acb57ce8b01) to [**`v0.3.0`**](https://github.com/golang/text/releases/tag/v0.3.0). -- Upgrade [`golang.org/x/time`](https://github.com/golang/time) from [**`time@c06e80d93`**](https://github.com/golang/time/commit/c06e80d9300e4443158a03817b8a8cb37d230320) to [**`time@fbb02b229`**](https://github.com/golang/time/commit/fbb02b2291d28baffd63558aa44b4b56f178d650). -- Upgrade [`github.com/golang/protobuf`](https://github.com/golang/protobuf/releases) from [**`golang/protobuf@1e59b77b5`**](https://github.com/golang/protobuf/commit/1e59b77b52bf8e4b449a57e6f79f21226d571845) to [**`v1.3.2`**](https://github.com/golang/protobuf/releases/tag/v1.3.2). -- Upgrade [`gopkg.in/yaml.v2`](https://github.com/go-yaml/yaml/releases) from [**`yaml@cd8b52f82`**](https://github.com/go-yaml/yaml/commit/cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b) to [**`yaml@5420a8b67`**](https://github.com/go-yaml/yaml/commit/5420a8b6744d3b0345ab293f6fcba19c978f1183). -- Upgrade [`github.com/dgrijalva/jwt-go`](https://github.com/dgrijalva/jwt-go/releases) from [**`v3.0.0`**](https://github.com/dgrijalva/jwt-go/releases/tag/v3.0.0) to [**`v3.2.0`**](https://github.com/dgrijalva/jwt-go/releases/tag/v3.2.0). -- Upgrade [`github.com/soheilhy/cmux`](https://github.com/soheilhy/cmux/releases) from [**`v0.1.3`**](https://github.com/soheilhy/cmux/releases/tag/v0.1.3) to [**`v0.1.4`**](https://github.com/soheilhy/cmux/releases/tag/v0.1.4). -- Upgrade [`github.com/google/btree`](https://github.com/google/btree/releases) from [**`google/btree@925471ac9`**](https://github.com/google/btree/commit/925471ac9e2131377a91e1595defec898166fe49) to [**`v1.0.0`**](https://github.com/google/btree/releases/tag/v1.0.0). -- Upgrade [`github.com/spf13/cobra`](https://github.com/spf13/cobra/releases) from [**`spf13/cobra@1c44ec8d3`**](https://github.com/spf13/cobra/commit/1c44ec8d3f1552cac48999f9306da23c4d8a288b) to [**`v0.0.3`**](https://github.com/spf13/cobra/releases/tag/v0.0.3). -- Upgrade [`github.com/spf13/pflag`](https://github.com/spf13/pflag/releases) from [**`v1.0.0`**](https://github.com/spf13/pflag/releases/tag/v1.0.0) to [**`spf13/pflag@1ce0cc6db`**](https://github.com/spf13/pflag/commit/1ce0cc6db4029d97571db82f85092fccedb572ce). -- Upgrade [`github.com/coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) from [**`v15`**](https://github.com/coreos/go-systemd/releases/tag/v15) to [**`v17`**](https://github.com/coreos/go-systemd/releases/tag/v17). -- Upgrade [`github.com/prometheus/client_golang`](https://github.com/prometheus/client_golang/releases) from [**``prometheus/client_golang@5cec1d042``**](https://github.com/prometheus/client_golang/commit/5cec1d0429b02e4323e042eb04dafdb079ddf568) to [**`v1.0.0`**](https://github.com/prometheus/client_golang/releases/tag/v1.0.0). -- Upgrade [`github.com/grpc-ecosystem/go-grpc-prometheus`](https://github.com/grpc-ecosystem/go-grpc-prometheus/releases) from [**``grpc-ecosystem/go-grpc-prometheus@0dafe0d49``**](https://github.com/grpc-ecosystem/go-grpc-prometheus/commit/0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7) to [**`v1.2.0`**](https://github.com/grpc-ecosystem/go-grpc-prometheus/releases/tag/v1.2.0). -- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.3.1`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.1) to [**`v1.4.1`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.4.1). -- Migrate [`github.com/kr/pty`](https://github.com/kr/pty/releases) to [**`github.com/creack/pty`**](https://github.com/creack/pty/releases/tag/v1.1.7), as the later has replaced the original module. -- Upgrade [`github.com/gogo/protobuf`](https://github.com/gogo/protobuf/releases) from [**`v1.0.0`**](https://github.com/gogo/protobuf/releases/tag/v1.0.0) to [**`v1.2.1`**](https://github.com/gogo/protobuf/releases/tag/v1.2.1). - -### Metrics, Monitoring - -See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. -- Add [`etcd_network_active_peers`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric. - - Let's say `"7339c4e5e833c029"` server `/metrics` returns `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 1`. This indicates that the local node `"7339c4e5e833c029"` currently has two active remote peers `"729934363faa4a24"` and `"b548c2511513015"` in a 3-node cluster. If the node `"b548c2511513015"` is down, the local node `"7339c4e5e833c029"` will show `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 0`. -- Add [`etcd_network_disconnected_peers_total`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric. - - If a remote peer `"b548c2511513015"` is down, the local node `"7339c4e5e833c029"` server `/metrics` would return `etcd_network_disconnected_peers_total{Local="7339c4e5e833c029",Remote="b548c2511513015"} 1`, while active peer metrics will show `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 0`. -- Add [`etcd_network_server_stream_failures_total`](https://github.com/etcd-io/etcd/pull/9760) Prometheus metric. - - e.g. `etcd_network_server_stream_failures_total{API="lease-keepalive",Type="receive"} 1` - - e.g. `etcd_network_server_stream_failures_total{API="watch",Type="receive"} 1` -- Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - - Previously, it only samples the TCP connection for snapshot messages. -- Increase [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric histogram upper-bound. - - Previously, highest bucket only collects requests taking 0.8192 seconds or more. - - Now, highest buckets collect 0.8192 seconds, 1.6384 seconds, and 3.2768 seconds or more. -- Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. -- Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. -- Add [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/10257) Prometheus metric. -- Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). -- Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. -- Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. -- Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. -- Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. -- Add [`etcd_server_slow_apply_total`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. -- Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. -- Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. -- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. -- Add [`etcd_mvcc_db_open_read_transactions`](https://github.com/etcd-io/etcd/pull/10523/commits/ad80752715aaed449629369687c5fd30eb1bda76) Prometheus metric. -- Add [`etcd_snap_fsync_duration_seconds`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric. -- Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. -- Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. -- Add [`etcd_mvcc_hash_rev_duration_seconds`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. -- Add [`etcd_debugging_disk_backend_commit_rebalance_duration_seconds`](https://github.com/etcd-io/etcd/pull/9834) Prometheus metric. -- Add [`etcd_debugging_disk_backend_commit_spill_duration_seconds`](https://github.com/etcd-io/etcd/pull/9834) Prometheus metric. -- Add [`etcd_debugging_disk_backend_commit_write_duration_seconds`](https://github.com/etcd-io/etcd/pull/9834) Prometheus metric. -- Add [`etcd_debugging_lease_granted_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. -- Add [`etcd_debugging_lease_revoked_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. -- Add [`etcd_debugging_lease_renewed_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. -- Add [`etcd_debugging_lease_ttl_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. -- Add [`etcd_network_snapshot_send_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. -- Add [`etcd_network_snapshot_receive_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. -- Add [`etcd_server_snapshot_apply_in_progress_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. -- Add [`etcd_server_is_learner`](https://github.com/etcd-io/etcd/pull/10731) Prometheus metric. -- Add [`etcd_server_learner_promote_failures`](https://github.com/etcd-io/etcd/pull/10731) Prometheus metric. -- Add [`etcd_server_learner_promote_successes`](https://github.com/etcd-io/etcd/pull/10731) Prometheus metric. -- Increase [`etcd_debugging_mvcc_index_compaction_pause_duration_milliseconds`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric histogram upper-bound. - - Previously, highest bucket only collects requests taking 1.024 seconds or more. - - Now, highest buckets collect 1.024 seconds, 2.048 seconds, and 4.096 seconds or more. -- Fix missing [`etcd_network_peer_sent_failures_total`](https://github.com/etcd-io/etcd/pull/9437) Prometheus metric count. -- Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/9557) Prometheus metric. -- Fix [race conditions in v2 server stat collecting](https://github.com/etcd-io/etcd/pull/9562). -- Change [gRPC proxy to expose etcd server endpoint /metrics](https://github.com/etcd-io/etcd/pull/10618). - - The metrics that were exposed via the proxy were not etcd server members but instead the proxy itself. -- Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). -- Deprecating `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) instead. -- Deprecating `etcd_debugging_mvcc_put_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_put_total`](https://github.com/etcd-io/etcd/pull/10962) instead. -- Deprecating `etcd_debugging_mvcc_delete_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_delete_total`](https://github.com/etcd-io/etcd/pull/10962) instead. -- Deprecating `etcd_debugging_mvcc_range_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_range_total`](https://github.com/etcd-io/etcd/pull/10968) instead. -- Deprecating `etcd_debugging_mvcc_txn_total`Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_txn_total`](https://github.com/etcd-io/etcd/pull/10968) instead. - -### Security, Authentication - -See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - -- Support TLS cipher suite whitelisting. - - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - - TLS handshake fails when client hello is requested with invalid cipher suites. - - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - - If empty, Go auto-populates the list. -- Add [`etcd --host-whitelist`](https://github.com/etcd-io/etcd/pull/9372) flag, [`etcdserver.Config.HostWhitelist`](https://github.com/etcd-io/etcd/pull/9372), and [`embed.Config.HostWhitelist`](https://github.com/etcd-io/etcd/pull/9372), to prevent ["DNS Rebinding"](https://en.wikipedia.org/wiki/DNS_rebinding) attack. - - Any website can simply create an authorized DNS name, and direct DNS to `"localhost"` (or any other address). Then, all HTTP endpoints of etcd server listening on `"localhost"` becomes accessible, thus vulnerable to [DNS rebinding attacks (CVE-2018-5702)](https://bugs.chromium.org/p/project-zero/issues/detail?id=1447#c2). - - Client origin enforce policy works as follow: - - If client connection is secure via HTTPS, allow any hostnames.. - - If client connection is not secure and `"HostWhitelist"` is not empty, only allow HTTP requests whose Host field is listed in whitelist. - - By default, `"HostWhitelist"` is `"*"`, which means insecure server allows all client HTTP requests. - - Note that the client origin policy is enforced whether authentication is enabled or not, for tighter controls. - - When specifying hostnames, loopback addresses are not added automatically. To allow loopback interfaces, add them to whitelist manually (e.g. `"localhost"`, `"127.0.0.1"`, etc.). - - e.g. `etcd --host-whitelist example.com`, then the server will reject all HTTP requests whose Host field is not `example.com` (also rejects requests to `"localhost"`). -- Support [`etcd --cors`](https://github.com/etcd-io/etcd/pull/9490) in v3 HTTP requests (gRPC gateway). -- Support [`ttl` field for `etcd` Authentication JWT token](https://github.com/etcd-io/etcd/pull/8302). - - e.g. `etcd --auth-token jwt,pub-key=,priv-key=,sign-method=,ttl=5m`. -- Allow empty token provider in [`etcdserver.ServerConfig.AuthToken`](https://github.com/etcd-io/etcd/pull/9369). -- Fix [TLS reload](https://github.com/etcd-io/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/etcd-io/etcd/issues/9541). - - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets. - - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/etcd-io/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online. - - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs). - -### etcd server - -- Add [`rpctypes.ErrLeaderChanged`](https://github.com/etcd-io/etcd/pull/10094). - - Now linearizable requests with read index would fail fast when there is a leadership change, instead of waiting until context timeout. -- Add [`etcd --initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - - Now, this can be disabled by setting `etcd --initial-election-tick-advance=false`. - - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `etcd --initial-election-tick-advance` at the cost of slow initial bootstrap. - - If single-node, it advances ticks regardless. - - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). -- Add [`etcd --pre-vote`](https://github.com/etcd-io/etcd/pull/9352) flag to enable to run an additional Raft election phase. - - For instance, a flaky(or rejoining) member may drop in and out, and start campaign. This member will end up with a higher term, and ignore all incoming messages with lower term. In this case, a new leader eventually need to get elected, thus disruptive to cluster availability. Raft implements Pre-Vote phase to prevent this kind of disruptions. If enabled, Raft runs an additional phase of election to check if pre-candidate can get enough votes to win an election. - - `etcd --pre-vote=false` by default. - - v3.5 will enable `etcd --pre-vote=true` by default. -- Add `etcd --experimental-compaction-batch-limit` to [sets the maximum revisions deleted in each compaction batch](https://github.com/etcd-io/etcd/pull/11034). -- Reduced default compaction batch size from 10k revisions to 1k revisions to improve p99 latency during compactions and reduced wait between compactions from 100ms to 10ms. -- Add [`etcd --discovery-srv-name`](https://github.com/etcd-io/etcd/pull/8690) flag to support custom DNS SRV name with discovery. - - If not given, etcd queries `_etcd-server-ssl._tcp.[YOUR_HOST]` and `_etcd-server._tcp.[YOUR_HOST]`. - - If `etcd --discovery-srv-name="foo"`, then query `_etcd-server-ssl-foo._tcp.[YOUR_HOST]` and `_etcd-server-foo._tcp.[YOUR_HOST]`. - - Useful for operating multiple etcd clusters under the same domain. -- Support TLS cipher suite whitelisting. - - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - - TLS handshake fails when client hello is requested with invalid cipher suites. - - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - - If empty, Go auto-populates the list. -- Support [`etcd --cors`](https://github.com/etcd-io/etcd/pull/9490) in v3 HTTP requests (gRPC gateway). -- Rename [`etcd --log-output` to `etcd --log-outputs`](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. - - **`etcd --log-output` will be deprecated in v3.5**. -- Add [`etcd --logger`](https://github.com/etcd-io/etcd/pull/9572) flag to support [structured logger and multiple log outputs](https://github.com/etcd-io/etcd/issues/9438) in server-side. - - **`etcd --logger=capnslog` will be deprecated in v3.5**. - - Main motivation is to promote automated etcd monitoring, rather than looking back server logs when it starts breaking. Future development will make etcd log as few as possible, and make etcd easier to monitor with metrics and alerts. - - `etcd --logger=capnslog --log-outputs=default` is the default setting and same as previous etcd server logging format. - - `etcd --logger=zap --log-outputs=default` is not supported when `etcd --logger=zap`. - - Use `etcd --logger=zap --log-outputs=stderr` instead. - - Or, use `etcd --logger=zap --log-outputs=systemd/journal` to send logs to the local systemd journal. - - Previously, if etcd parent process ID (PPID) is 1 (e.g. run with systemd), `etcd --logger=capnslog --log-outputs=default` redirects server logs to local systemd journal. And if write to journald fails, it writes to `os.Stderr` as a fallback. - - However, even with PPID 1, it can fail to dial systemd journal (e.g. run embedded etcd with Docker container). Then, [every single log write will fail](https://github.com/etcd-io/etcd/pull/9729) and fall back to `os.Stderr`, which is inefficient. - - To avoid this problem, systemd journal logging must be configured manually. - - `etcd --logger=zap --log-outputs=stderr` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to `os.Stderr`. Use this to override journald log redirects. - - `etcd --logger=zap --log-outputs=stdout` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to `os.Stdout` Use this to override journald log redirects. - - `etcd --logger=zap --log-outputs=a.log` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to the specified file `a.log`. - - `etcd --logger=zap --log-outputs=a.log,b.log,c.log,stdout` [writes server logs to multiple files `a.log`, `b.log` and `c.log` at the same time](https://github.com/etcd-io/etcd/pull/9579) and outputs to `os.Stderr`, in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig). - - `etcd --logger=zap --log-outputs=/dev/null` will discard all server logs. -- Add [`etcd --log-level`](https://github.com/etcd-io/etcd/pull/10947) flag to support log level. - - v3.5 will deprecate `etcd --debug` flag in favor of `etcd --log-level=debug`. -- Add [`etcd --backend-batch-limit`](https://github.com/etcd-io/etcd/pull/10283) flag. -- Add [`etcd --backend-batch-interval`](https://github.com/etcd-io/etcd/pull/10283) flag. -- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9281). - - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. -- Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - - Now, this server-side panic has been fixed. -- Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/etcd-io/etcd/pull/9379). - - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server. - - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`. -- Fix [revision-based compaction retention parsing](https://github.com/etcd-io/etcd/pull/9339). - - Previously, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` was [translated to revision retention 3600000000000](https://github.com/etcd-io/etcd/issues/9337). - - Now, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` is correctly parsed as revision retention 1. -- Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/etcd-io/etcd/pull/9399). - - `TTL` parameter to `Grant` request is unit of second. - - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/etcd-io/etcd/issues/9374). - - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! -- Fix [expired lease revoke](https://github.com/etcd-io/etcd/pull/10693). - - Fix ["the key is not deleted when the bound lease expires"](https://github.com/etcd-io/etcd/issues/10686). -- Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/etcd-io/etcd/pull/9347). -- Allow [non-WAL files in `etcd --wal-dir` directory](https://github.com/etcd-io/etcd/pull/9743). - - Previously, existing files such as [`lost+found`](https://github.com/etcd-io/etcd/issues/7287) in WAL directory prevent etcd server boot. - - Now, WAL directory that contains only `lost+found` or a file that's not suffixed with `.wal` is considered non-initialized. -- Fix [`ETCD_CONFIG_FILE` env variable parsing in `etcd`](https://github.com/etcd-io/etcd/pull/10762). -- Fix [race condition in `rafthttp` transport pause/resume](https://github.com/etcd-io/etcd/pull/10826). -- Fix [server crash from creating an empty role](https://github.com/etcd-io/etcd/pull/10907). - - Previously, creating a role with an empty name crashed etcd server with an error code `Unavailable`. - - Now, creating a role with an empty name is not allowed with an error code `InvalidArgument`. - -### API - -- Add `isLearner` field to `etcdserverpb.Member`, `etcdserverpb.MemberAddRequest` and `etcdserverpb.StatusResponse` as part of [raft learner implementation](https://github.com/etcd-io/etcd/pull/10725). -- Add `MemberPromote` rpc to `etcdserverpb.Cluster` interface and the corresponding `MemberPromoteRequest` and `MemberPromoteResponse` as part of [raft learner implementation](https://github.com/etcd-io/etcd/pull/10725). -- Add [`snapshot`](https://github.com/etcd-io/etcd/pull/9118) package for snapshot restore/save operations (see [`godoc.org/github.com/etcd/clientv3/snapshot`](https://godoc.org/github.com/coreos/etcd/clientv3/snapshot) for more). -- Add [`watch_id` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9065) to allow user-provided watch ID to `mvcc`. - - Corresponding `watch_id` is returned via `etcdserverpb.WatchResponse`, if any. -- Add [`fragment` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9291) to request etcd server to [split watch events](https://github.com/etcd-io/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes. - - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes. - - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit. - - Useful when client-side has limited bandwidths. - - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client. - - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`. - - Client must implement fragmented watch event merge (which `clientv3` does in etcd v3.4). -- Add [`raftAppliedIndex` field to `etcdserverpb.StatusResponse`](https://github.com/etcd-io/etcd/pull/9176) for current Raft applied index. -- Add [`errors` field to `etcdserverpb.StatusResponse`](https://github.com/etcd-io/etcd/pull/9206) for server-side error. - - e.g. `"etcdserver: no leader", "NOSPACE", "CORRUPT"` -- Add [`dbSizeInUse` field to `etcdserverpb.StatusResponse`](https://github.com/etcd-io/etcd/pull/9256) for actual DB size after compaction. -- Add [`WatchRequest.WatchProgressRequest`](https://github.com/etcd-io/etcd/pull/9869). - - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - - Think of it as `WithProgressNotify` that can be triggered manually. - -Note: **v3.5 will deprecate `etcd --log-package-levels` flag for `capnslog`**; `etcd --logger=zap --log-outputs=stderr` will the default. **v3.5 will deprecate `[CLIENT-URL]/config/local/log` endpoint.** - -### Package `embed` - -- Add [`embed.Config.CipherSuites`](https://github.com/etcd-io/etcd/pull/9801) to specify a list of supported cipher suites for TLS handshake between client/server and peers. - - If empty, Go auto-populates the list. - - Both `embed.Config.ClientTLSInfo.CipherSuites` and `embed.Config.CipherSuites` cannot be non-empty at the same time. - - If not empty, specify either `embed.Config.ClientTLSInfo.CipherSuites` or `embed.Config.CipherSuites`. -- Add [`embed.Config.InitialElectionTickAdvance`](https://github.com/etcd-io/etcd/pull/9591) to enable/disable initial election tick fast-forward. - - `embed.NewConfig()` would return `*embed.Config` with `InitialElectionTickAdvance` as true by default. -- Define [`embed.CompactorModePeriodic`](https://godoc.org/github.com/etcd-io/etcd/embed#pkg-variables) for `compactor.ModePeriodic`. -- Define [`embed.CompactorModeRevision`](https://godoc.org/github.com/etcd-io/etcd/embed#pkg-variables) for `compactor.ModeRevision`. -- Change [`embed.Config.CorsInfo` in `*cors.CORSInfo` type to `embed.Config.CORS` in `map[string]struct{}` type](https://github.com/etcd-io/etcd/pull/9490). -- Remove [`embed.Config.SetupLogging`](https://github.com/etcd-io/etcd/pull/9572). - - Now logger is set up automatically based on [`embed.Config.Logger`, `embed.Config.LogOutputs`, `embed.Config.Debug` fields](https://github.com/etcd-io/etcd/pull/9572). -- Add [`embed.Config.Logger`](https://github.com/etcd-io/etcd/pull/9518) to support [structured logger `zap`](https://github.com/uber-go/zap) in server-side. -- Add [`embed.Config.LogLevel`](https://github.com/etcd-io/etcd/pull/10947). -- Rename `embed.Config.SnapCount` field to [`embed.Config.SnapshotCount`](https://github.com/etcd-io/etcd/pull/9745), to be consistent with the flag name `etcd --snapshot-count`. -- Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. -- Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/etcd-io/etcd/pull/9579) to support multiple log outputs. -- Add [`embed.Config.BackendBatchLimit`](https://github.com/etcd-io/etcd/pull/10283) field. -- Add [`embed.Config.BackendBatchInterval`](https://github.com/etcd-io/etcd/pull/10283) field. -- Make [`embed.DefaultEnableV2` `false` default](https://github.com/etcd-io/etcd/pull/10935). - -### Package `pkg/adt` - -- Change [`pkg/adt.IntervalTree` from `struct` to `interface`](https://github.com/etcd-io/etcd/pull/10959). - - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). -- Improve [`pkg/adt.IntervalTree` test coverage](https://github.com/etcd-io/etcd/pull/10959). - - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). -- Fix [Red-Black tree to maintain black-height property](https://github.com/etcd-io/etcd/pull/10978). - - Previously, delete operation violates [black-height property](https://github.com/etcd-io/etcd/issues/10965). - -### Package `integration` - -- Add [`CLUSTER_DEBUG` to enable test cluster logging](https://github.com/etcd-io/etcd/pull/9678). - - Deprecated `capnslog` in integration tests. - -### client v3 - -- Add [`MemberAddAsLearner`](https://github.com/etcd-io/etcd/pull/10725) to `Clientv3.Cluster` interface. This API is used to add a learner member to etcd cluster. -- Add [`MemberPromote`](https://github.com/etcd-io/etcd/pull/10727) to `Clientv3.Cluster` interface. This API is used to promote a learner member in etcd cluster. -- Client may receive [`rpctypes.ErrLeaderChanged`](https://github.com/etcd-io/etcd/pull/10094) from server. - - Now linearizable requests with read index would fail fast when there is a leadership change, instead of waiting until context timeout. -- Add [`WithFragment` `OpOption`](https://github.com/etcd-io/etcd/pull/9291) to support [watch events fragmentation](https://github.com/etcd-io/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes. - - Watch fragmentation is disabled by default. - - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes. - - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit. - - Useful when client-side has limited bandwidths. - - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client. - - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`. -- Add [`Watcher.RequestProgress` method](https://github.com/etcd-io/etcd/pull/9869). - - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - - Think of it as `WithProgressNotify` that can be triggered manually. -- Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. -- Change [snapshot file permissions](https://github.com/etcd-io/etcd/pull/9977): On Linux, the snapshot file changes from readable by all (mode 0644) to readable by the user only (mode 0600). -- Client may choose to send keepalive pings to server using [`PermitWithoutStream`](https://github.com/etcd-io/etcd/pull/10146). - - By setting `PermitWithoutStream` to true, client can send keepalive pings to server without any active streams(RPCs). In other words, it allows sending keepalive pings with unary or simple RPC calls. - - `PermitWithoutStream` is set to false by default. -- Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. -- Fix [`(*Client).Endpoints()` method race condition](https://github.com/etcd-io/etcd/pull/10595). -- Deprecated [`grpc.ErrClientConnClosing`](https://github.com/etcd-io/etcd/pull/10981). - - `clientv3` and `proxy/grpcproxy` now does not return `grpc.ErrClientConnClosing`. - - `grpc.ErrClientConnClosing` has been [deprecated in gRPC >= 1.10](https://github.com/grpc/grpc-go/pull/1854). - - Use `clientv3.IsConnCanceled(error)` or `google.golang.org/grpc/status.FromError(error)` instead. - -### etcdctl v3 - -- Make [`ETCDCTL_API=3 etcdctl` default](https://github.com/etcd-io/etcd/issues/9600). - - Now, `etcdctl set foo bar` must be `ETCDCTL_API=2 etcdctl set foo bar`. - - Now, `ETCDCTL_API=3 etcdctl put foo bar` could be just `etcdctl put foo bar`. -- Add [`etcdctl member add --learner` and `etcdctl member promote`](https://github.com/etcd-io/etcd/pull/10725) to add and promote raft learner member in etcd cluster. -- Add [`etcdctl --password`](https://github.com/etcd-io/etcd/pull/9730) flag. - - To support [`:` character in user name](https://github.com/etcd-io/etcd/issues/9691). - - e.g. `etcdctl --user user --password password get foo` -- Add [`etcdctl user add --new-user-password`](https://github.com/etcd-io/etcd/pull/9730) flag. -- Add [`etcdctl check datascale`](https://github.com/etcd-io/etcd/pull/9185) command. -- Add [`etcdctl check datascale --auto-compact, --auto-defrag`](https://github.com/etcd-io/etcd/pull/9351) flags. -- Add [`etcdctl check perf --auto-compact, --auto-defrag`](https://github.com/etcd-io/etcd/pull/9330) flags. -- Add [`etcdctl defrag --cluster`](https://github.com/etcd-io/etcd/pull/9390) flag. -- Add ["raft applied index" field to `endpoint status`](https://github.com/etcd-io/etcd/pull/9176). -- Add ["errors" field to `endpoint status`](https://github.com/etcd-io/etcd/pull/9206). -- Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). -- Add [missing newline in `etcdctl endpoint health`](https://github.com/etcd-io/etcd/pull/10793). -- Fix [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/etcd-io/etcd/pull/9688) parsing. - - Previously, `ETCDCTL_API=3 etcdctl watch foo -- echo watch event received` panicked. -- Fix [`etcdctl move-leader` command for TLS-enabled endpoints](https://github.com/etcd-io/etcd/pull/9807). -- Add [`progress` command to `etcdctl watch --interactive`](https://github.com/etcd-io/etcd/pull/9869). - - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - - Think of it as `WithProgressNotify` that can be triggered manually. -- Add [timeout](https://github.com/etcd-io/etcd/pull/10301) to `etcdctl snapshot - save`. - - User can specify timeout of `etcdctl snapshot save` command using flag `--command-timeout`. - - Fix etcdctl to [strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) - -### gRPC proxy - -- Fix [etcd server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - - Especially, gRPC proxy was affected, since it detects a leader loss with a key `"proxy-namespace__lostleader"` and a watch revision `"int64(math.MaxInt64 - 2)"`. - - Now, this server-side panic has been fixed. -- Fix [memory leak in cache layer](https://github.com/etcd-io/etcd/pull/10327). -- Change [gRPC proxy to expose etcd server endpoint /metrics](https://github.com/etcd-io/etcd/pull/10618). - - The metrics that were exposed via the proxy were not etcd server members but instead the proxy itself. - -### gRPC gateway - -- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3beta` with [`/v3`](https://github.com/etcd-io/etcd/pull/9298). - - Deprecated [`/v3alpha`](https://github.com/etcd-io/etcd/pull/9298). - - To deprecate [`/v3beta`](https://github.com/etcd-io/etcd/issues/9189) in v3.5. - - In v3.4, `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. -- Add API endpoints [`/{v3beta,v3}/lease/leases, /{v3beta,v3}/lease/revoke, /{v3beta,v3}/lease/timetolive`](https://github.com/etcd-io/etcd/pull/9450). - - To deprecate [`/{v3beta,v3}/kv/lease/leases, /{v3beta,v3}/kv/lease/revoke, /{v3beta,v3}/kv/lease/timetolive`](https://github.com/etcd-io/etcd/issues/9430) in v3.5. -- Support [`etcd --cors`](https://github.com/etcd-io/etcd/pull/9490) in v3 HTTP requests (gRPC gateway). - -### Package `raft` - -- Fix [deadlock during PreVote migration process](https://github.com/etcd-io/etcd/pull/8525). -- Add [`raft.ErrProposalDropped`](https://github.com/etcd-io/etcd/pull/9067). - - Now [`(r *raft) Step` returns `raft.ErrProposalDropped`](https://github.com/etcd-io/etcd/pull/9137) if a proposal has been ignored. - - e.g. a node is removed from cluster, or [`raftpb.MsgProp` arrives at current leader while there is an ongoing leadership transfer](https://github.com/etcd-io/etcd/issues/8975). -- Improve [Raft `becomeLeader` and `stepLeader`](https://github.com/etcd-io/etcd/pull/9073) by keeping track of latest `pb.EntryConfChange` index. - - Previously record `pendingConf` boolean field scanning the entire tail of the log, which can delay hearbeat send. -- Fix [missing learner nodes on `(n *node) ApplyConfChange`](https://github.com/etcd-io/etcd/pull/9116). -- Add [`raft.Config.MaxUncommittedEntriesSize`](https://github.com/etcd-io/etcd/pull/10167) to limit the total size of the uncommitted entries in bytes. - - Once exceeded, raft returns `raft.ErrProposalDropped` error. - - Prevent [unbounded Raft log growth](https://github.com/cockroachdb/cockroach/issues/27772). - - There was a bug in [PR#10167](https://github.com/etcd-io/etcd/pull/10167) but fixed via [PR#10199](https://github.com/etcd-io/etcd/pull/10199). -- Add [`raft.Ready.CommittedEntries` pagination using `raft.Config.MaxSizePerMsg`](https://github.com/etcd-io/etcd/pull/9982). - - This prevents out-of-memory errors if the raft log has become very large and commits all at once. - - Fix [correctness bug in CommittedEntries pagination](https://github.com/etcd-io/etcd/pull/10063). -- Optimize [message send flow control](https://github.com/etcd-io/etcd/pull/9985). - - Leader now sends more append entries if it has more non-empty entries to send after updating flow control information. - - Now, Raft allows multiple in-flight append messages. -- Optimize [memory allocation when boxing slice in `maybeCommit`](https://github.com/etcd-io/etcd/pull/10679). - - By boxing a heap-allocated slice header instead of the slice header on the stack, we can avoid an allocation when passing through the sort.Interface interface. -- Avoid [memory allocation in Raft entry `String` method](https://github.com/etcd-io/etcd/pull/10680). -- Avoid [multiple memory allocations when merging stable and unstable log](https://github.com/etcd-io/etcd/pull/10684). -- Extract [progress tracking into own component](https://github.com/etcd-io/etcd/pull/10683). - - Add [package `raft/tracker`](https://github.com/etcd-io/etcd/pull/10807). - - Optimize [string representation of `Progress`](https://github.com/etcd-io/etcd/pull/10882). -- Make [relationship between `node` and `RawNode` explicit](https://github.com/etcd-io/etcd/pull/10803). -- Prevent [learners from becoming leader](https://github.com/etcd-io/etcd/pull/10822). -- Add [package `raft/quorum` to reason about committed indexes as well as vote outcomes for both majority and joint quorums](https://github.com/etcd-io/etcd/pull/10779). - - Bundle [Voters and Learner into `raft/tracker.Config` struct](https://github.com/etcd-io/etcd/pull/10865). -- Use [membership sets in progress tracking](https://github.com/etcd-io/etcd/pull/10779). -- Implement [joint quorum computation](https://github.com/etcd-io/etcd/pull/10779). -- Refactor [`raft/node.go` to centralize configuration change application](https://github.com/etcd-io/etcd/pull/10865). -- Allow [voter to become learner through snapshot](https://github.com/etcd-io/etcd/pull/10864). -- Add [package `raft/confchange` to internally support joint consensus](https://github.com/etcd-io/etcd/pull/10779). -- Use [`RawNode` for node's event loop](https://github.com/etcd-io/etcd/pull/10892). -- Add [`RawNode.Bootstrap` method](https://github.com/etcd-io/etcd/pull/10892). -- Add [`raftpb.ConfChangeV2` to use joint quorums](https://github.com/etcd-io/etcd/pull/10914). - - `raftpb.ConfChange` continues to work as today: it allows carrying out a single configuration change. A `pb.ConfChange` proposal gets added to the Raft log as such and is thus also observed by the app during Ready handling, and fed back to ApplyConfChange. - - `raftpb.ConfChangeV2` allows joint configuration changes but will continue to carry out configuration changes in "one phase" (i.e. without ever entering a joint config) when this is possible. - - `raftpb.ConfChangeV2` messages initiate configuration changes. They support both the simple "one at a time" membership change protocol and full Joint Consensus allowing for arbitrary changes in membership. -- Change [`raftpb.ConfState.Nodes` to `raftpb.ConfState.Voters`](https://github.com/etcd-io/etcd/pull/10914). -- Allow [learners to vote, but still learners do not count in quorum](https://github.com/etcd-io/etcd/pull/10998). - - necessary in the situation in which a learner has been promoted (i.e. is now a voter) but has not learned about this yet. -- Fix [restoring joint consensus](https://github.com/etcd-io/etcd/pull/11003). -- Visit [`Progress` in stable order](https://github.com/etcd-io/etcd/pull/11004). -- Proactively [probe newly added followers](https://github.com/etcd-io/etcd/pull/11037). - - The general expectation in `tracker.Progress.Next == c.LastIndex` is that the follower has no log at all (and will thus likely need a snapshot), though the app may have applied a snapshot out of band before adding the replica (thus making the first index the better choice). - - Previously, when the leader applied a new configuration that added voters, it would not immediately probe these voters, delaying when they would be caught up. - -### Package `wal` - -- Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). -- Fix [`wal` directory cleanup on creation failures](https://github.com/etcd-io/etcd/pull/10689). - -### Tooling - -- Add [`etcd-dump-logs --entry-type`](https://github.com/etcd-io/etcd/pull/9628) flag to support WAL log filtering by entry type. -- Add [`etcd-dump-logs --stream-decoder`](https://github.com/etcd-io/etcd/pull/9790) flag to support custom decoder. -- Add [`SHA256SUMS`](https://github.com/etcd-io/etcd/pull/11087) file to release assets. - - etcd maintainers are a distributed team, this change allows for releases to be cut and validation provided without requiring a signing key. - -### Go - -- Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). -- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - -### Dockerfile - -- [Rebase etcd image from Alpine to Debian](https://github.com/etcd-io/etcd/pull/10805) to improve security and maintenance effort for etcd release. - -
- diff --git a/CHANGELOG-3.5.md b/CHANGELOG-3.5.md deleted file mode 100644 index b60a08e5c79d..000000000000 --- a/CHANGELOG-3.5.md +++ /dev/null @@ -1,296 +0,0 @@ - - -Previous change logs can be found at [CHANGELOG-3.4](https://github.com/etcd-io/etcd/blob/main/CHANGELOG-3.4.md). - - -The minimum recommended etcd versions to run in **production** are 3.2.28+, 3.3.18+, and 3.4.2+. - - -
- - -## v3.5.0 (2021-06) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.5.0) and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/) for any breaking changes. - -- [v3.5.0](https://github.com/etcd-io/etcd/releases/tag/v3.5.0) (2020 TBD), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-rc.1...v3.5.0). -- [v3.5.0-rc.1](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-rc.1) (2020 TBD), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-rc.0...v3.5.0-rc.1). -- [v3.5.0-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-rc.0) (2020 TBD), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.5.0-rc.0). - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/).** - -### Breaking Changes - -- `go.etcd.io/etcd` Go packages have moved to `go.etcd.io/etcd/{api,pkg,raft,client,etcdctl,server,raft,tests}/v3` to follow the [Go modules](https://github.com/golang/go/wiki/Modules) conventions -- `go.etcd.io/clientv3/snapshot` SnapshotManager class have moved to `go.etcd.io/clientv3/etcdctl`. - The method `snapshot.Save` to download a snapshot from the remote server was preserved in 'go.etcd.io/clientv3/snapshot`. -- `go.etcd.io/client' package got migrated to 'go.etcd.io/client/v2'. -- Changed behavior of clienv3 API [MemberList](https://github.com/etcd-io/etcd/pull/11639). - - Previously, it is directly served with server's local data, which could be stale. - - Now, it is served with linearizable guarantee. If the server is disconnected from quorum, `MemberList` call will fail. -- [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) only supports [`/v3`](TODO) endpoint. - - Deprecated [`/v3beta`](https://github.com/etcd-io/etcd/pull/9298). - - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` does work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. -- **`etcd --experimental-enable-v2v3` flag remains experimental and to be deprecated.** - - v2 storage emulation feature will be deprecated in the next release. - - etcd 3.5 is the last version that supports V2 API. Flags `--enable-v2` and `--experimental-enable-v2v3` [are now deprecated](https://github.com/etcd-io/etcd/pull/) and will be removed in etcd v3.6 release. -- **`etcd --experimental-backend-bbolt-freelist-type` flag has been deprecated.** Use **`etcd --backend-bbolt-freelist-type`** instead. The default type is hashmap and it is stable now. -- **`etcd --debug` flag has been deprecated.** Use **`etcd --log-level=debug`** instead. -- Remove [`embed.Config.Debug`](https://github.com/etcd-io/etcd/pull/10947). -- **`etcd --log-output` flag has been deprecated.** Use **`etcd --log-outputs`** instead. -- **`etcd --logger=zap --log-outputs=stderr`** is now the default. -- **`etcd --logger=capnslog` flag value has been deprecated.** -- **`etcd --logger=zap --log-outputs=default` flag value is not supported.**. - - Use `etcd --logger=zap --log-outputs=stderr`. - - Or, use `etcd --logger=zap --log-outputs=systemd/journal` to send logs to the local systemd journal. - - Previously, if etcd parent process ID (PPID) is 1 (e.g. run with systemd), `etcd --logger=capnslog --log-outputs=default` redirects server logs to local systemd journal. And if write to journald fails, it writes to `os.Stderr` as a fallback. - - However, even with PPID 1, it can fail to dial systemd journal (e.g. run embedded etcd with Docker container). Then, [every single log write will fail](https://github.com/etcd-io/etcd/pull/9729) and fall back to `os.Stderr`, which is inefficient. - - To avoid this problem, systemd journal logging must be configured manually. -- **`etcd --log-outputs=stderr`** is now the default. -- **`etcd --log-package-levels` flag for `capnslog` has been deprecated.** Now, **`etcd --logger=zap --log-outputs=stderr`** is the default. -- **`[CLIENT-URL]/config/local/log` endpoint has been deprecated, as is `etcd --log-package-levels` flag.** - - `curl http://127.0.0.1:2379/config/local/log -XPUT -d '{"Level":"DEBUG"}'` won't work. - - Please use `etcd --logger=zap --log-outputs=stderr` instead. -- Deprecated `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric. Use `etcd_mvcc_db_total_size_in_bytes` instead. -- Deprecated `etcd_debugging_mvcc_put_total` Prometheus metric. Use `etcd_mvcc_put_total` instead. -- Deprecated `etcd_debugging_mvcc_delete_total` Prometheus metric. Use `etcd_mvcc_delete_total` instead. -- Deprecated `etcd_debugging_mvcc_txn_total` Prometheus metric. Use `etcd_mvcc_txn_total` instead. -- Deprecated `etcd_debugging_mvcc_range_total` Prometheus metric. Use `etcd_mvcc_range_total` instead. -- Main branch `/version` outputs `3.5.0-pre`, instead of `3.4.0+git`. -- Changed `proxy` package function signature to [support structured logger](https://github.com/etcd-io/etcd/pull/11614). - - Previously, `NewClusterProxy(c *clientv3.Client, advaddr string, prefix string) (pb.ClusterServer, <-chan struct{})`, now `NewClusterProxy(lg *zap.Logger, c *clientv3.Client, advaddr string, prefix string) (pb.ClusterServer, <-chan struct{})`. - - Previously, `Register(c *clientv3.Client, prefix string, addr string, ttl int)`, now `Register(lg *zap.Logger, c *clientv3.Client, prefix string, addr string, ttl int) <-chan struct{}`. - - Previously, `NewHandler(t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler`, now `NewHandler(lg *zap.Logger, t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler`. -- Changed `pkg/flags` function signature to [support structured logger](https://github.com/etcd-io/etcd/pull/11616). - - Previously, `SetFlagsFromEnv(prefix string, fs *flag.FlagSet) error`, now `SetFlagsFromEnv(lg *zap.Logger, prefix string, fs *flag.FlagSet) error`. - - Previously, `SetPflagsFromEnv(prefix string, fs *pflag.FlagSet) error`, now `SetPflagsFromEnv(lg *zap.Logger, prefix string, fs *pflag.FlagSet) error`. -- ClientV3 supports [grpc resolver API](https://github.com/etcd-io/etcd/blob/main/client/v3/naming/resolver/resolver.go). - - Endpoints can be managed using [endpoints.Manager](https://github.com/etcd-io/etcd/blob/main/client/v3/naming/endpoints/endpoints.go) - - Previously supported [GRPCResolver was decomissioned](https://github.com/etcd-io/etcd/pull/12675). Use [resolver](https://github.com/etcd-io/etcd/blob/main/client/v3/naming/resolver/resolver.go) instead. -- Turned on [--pre-vote by default](https://github.com/etcd-io/etcd/pull/12770). Should prevent disrupting RAFT leader by an individual member. -- [ETCD_CLIENT_DEBUG env](https://github.com/etcd-io/etcd/pull/12786): Now supports log levels (debug, info, warn, error, dpanic, panic, fatal). Only when set, overrides application-wide grpc logging settings. -- [Embed Etcd.Close()](https://github.com/etcd-io/etcd/pull/12828) needs to called exactly once and closes Etcd.Err() stream. -- [Embed Etcd does not override global/grpc logger](https://github.com/etcd-io/etcd/pull/12861) be default any longer. If desired, please call `embed.Config::SetupGlobalLoggers()` explicitly. -- [Embed Etcd custom logger should be configured using simpler builder `NewZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/12973). -- Client errors of `context cancelled` or `context deadline exceeded` are exposed as `codes.Canceled` and `codes.DeadlineExceeded`, instead of `codes.Unknown`. - - -### Storage format changes -- [WAL log's snapshots persists raftpb.ConfState](https://github.com/etcd-io/etcd/pull/12735) -- [Backend persists raftpb.ConfState](https://github.com/etcd-io/etcd/pull/12962) in the `meta` bucket `confState` key. -- [Backend persists applied term](https://github.com/etcd-io/etcd/pull/) in the `meta` bucket. -- Backend persists `downgrade` in the `cluster` bucket - -### Security - -- Add [`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256` and `TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256` to `etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/11864). -- Changed [the format of WAL entries related to auth for not keeping password as a plain text](https://github.com/etcd-io/etcd/pull/11943). -- Add third party [Security Audit Report](https://github.com/etcd-io/etcd/pull/12201). -- A [log warning](https://github.com/etcd-io/etcd/pull/12242) is added when etcd uses any existing directory that has a permission different than 700 on Linux and 777 on Windows. - -### Metrics, Monitoring - -See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - -Note that any `etcd_debugging_*` metrics are experimental and subject to change. - -- Deprecated `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric. Use `etcd_mvcc_db_total_size_in_bytes` instead. -- Deprecated `etcd_debugging_mvcc_put_total` Prometheus metric. Use `etcd_mvcc_put_total` instead. -- Deprecated `etcd_debugging_mvcc_delete_total` Prometheus metric. Use `etcd_mvcc_delete_total` instead. -- Deprecated `etcd_debugging_mvcc_txn_total` Prometheus metric. Use `etcd_mvcc_txn_total` instead. -- Deprecated `etcd_debugging_mvcc_range_total` Prometheus metric. Use `etcd_mvcc_range_total` instead. -- Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. -- Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. -- Change [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11254) Prometheus metrics to include only major and minor version. -- Add [`etcd_debugging_mvcc_total_put_size_in_bytes`](https://github.com/etcd-io/etcd/pull/11374) Prometheus metric. -- Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). -- Add [`etcd_debugging_auth_revision`](https://github.com/etcd-io/etcd/commit/f14d2a087f7b0fd6f7980b95b5e0b945109c95f3). -- Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). - -### etcd server - - - Add [`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256` and `TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256` to `etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/11864). - - Automatically [create parent directory if it does not exist](https://github.com/etcd-io/etcd/pull/9626) (fix [issue#9609](https://github.com/etcd-io/etcd/issues/9609)). - - v4.0 will configure `etcd --enable-v2=true --enable-v2v3=/aaa` to enable v2 API server that is backed by **v3 storage**. -- [`etcd --backend-bbolt-freelist-type`] flag is now stable. - - `etcd --experimental-backend-bbolt-freelist-type` has been deprecated. -- Support [downgrade API](https://github.com/etcd-io/etcd/pull/11715). -- Deprecate v2 apply on cluster version. [Use v3 request to set cluster version and recover cluster version from v3 backend](https://github.com/etcd-io/etcd/pull/11427). -- [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). -- Fix [quorum protection logic when promoting a learner](https://github.com/etcd-io/etcd/pull/11640). -- Improve [peer corruption checker](https://github.com/etcd-io/etcd/pull/11621) to work when peer mTLS is enabled. -- Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). -- Log [successful etcd server-side health check in debug level](https://github.com/etcd-io/etcd/pull/12677). -- Improve [compaction performance when latest index is greater than 1-million](https://github.com/etcd-io/etcd/pull/11734). -- [Refactor consistentindex](https://github.com/etcd-io/etcd/pull/11699). -- [Add log when etcdserver failed to apply command](https://github.com/etcd-io/etcd/pull/11670). -- Improve [count-only range performance](https://github.com/etcd-io/etcd/pull/11771). -- Remove [redundant storage restore operation to shorten the startup time](https://github.com/etcd-io/etcd/pull/11779). - - With 40 million key test data,it can shorten the startup time from 5 min to 2.5 min. -- [Fix deadlock bug in mvcc](https://github.com/etcd-io/etcd/pull/11817). -- Fix [inconsistency between WAL and server snapshot](https://github.com/etcd-io/etcd/pull/11888). - - Previously, server restore fails if it had crashed after persisting raft hard state but before saving snapshot. - - See https://github.com/etcd-io/etcd/issues/10219 for more. - - Add [missing CRC checksum check in WAL validate method otherwise causes panic](https://github.com/etcd-io/etcd/pull/11924). - - See https://github.com/etcd-io/etcd/issues/11918. -- Improve logging around snapshot send and receive. -- [Push down RangeOptions.limit argv into index tree to reduce memory overhead](https://github.com/etcd-io/etcd/pull/11990). -- Add [reason field for /health response](https://github.com/etcd-io/etcd/pull/11983). -- Add [exclude alarms from health check conditionally](https://github.com/etcd-io/etcd/pull/12880). -- Add [`etcd --unsafe-no-fsync`](https://github.com/etcd-io/etcd/pull/11946) flag. - - Setting the flag disables all uses of fsync, which is unsafe and will cause data loss. This flag makes it possible to run an etcd node for testing and development without placing lots of load on the file system. -- Add [`etcd --auth-token-ttl`](https://github.com/etcd-io/etcd/pull/11980) flag to customize `simpleTokenTTL` settings. -- Improve [`runtime.FDUsage` call pattern to reduce objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). -- Improve [mvcc.watchResponse channel Memory Usage](https://github.com/etcd-io/etcd/pull/11987). -- Log [expensive request info in UnaryInterceptor](https://github.com/etcd-io/etcd/pull/12086). -- [Fix invalid Go type in etcdserverpb](https://github.com/etcd-io/etcd/pull/12000). -- [Improve healthcheck by using v3 range request and its corresponding timeout](https://github.com/etcd-io/etcd/pull/12195). -- Add [`etcd --experimental-watch-progress-notify-interval`](https://github.com/etcd-io/etcd/pull/12216) flag to make watch progress notify interval configurable. -- Fix [server panic in slow writes warnings](https://github.com/etcd-io/etcd/issues/12197). - - Fixed via [PR#12238](https://github.com/etcd-io/etcd/pull/12238). -- [Fix server panic](https://github.com/etcd-io/etcd/pull/12288) when force-new-cluster flag is enabled in a cluster which had learner node. -- Add [`etcd --self-signed-cert-validity`](https://github.com/etcd-io/etcd/pull/12429) flag to support setting certificate expiration time. - - Notice, certificates generated by etcd are valid for 1 year by default when specifying the auto-tls or peer-auto-tls option. -- Add [`etcd --experimental-warning-apply-duration`](https://github.com/etcd-io/etcd/pull/12448) flag which allows apply duration threshold to be configurable. -- Add [`etcd --experimental-memory-mlock`](https://github.com/etcd-io/etcd/pull/TODO) flag which prevents etcd memory pages to be swapped out. -- Add [`etcd --socket-reuse-port`](https://github.com/etcd-io/etcd/pull/12702) flag - - Setting this flag enables `SO_REUSEPORT` which allows rebind of a port already in use. User should take caution when using this flag to ensure flock is properly enforced. -- Add [`etcd --socket-reuse-address`](https://github.com/etcd-io/etcd/pull/12702) flag - - Setting this flag enables `SO_REUSEADDR` which allows binding to an address in `TIME_WAIT` state, improving etcd restart time. -- Reduce [around 30% memory allocation by logging range response size without marshal](https://github.com/etcd-io/etcd/pull/12871). -- `ETCD_VERIFY="all"` environment triggers [additional verification of consistency](https://github.com/etcd-io/etcd/pull/) of etcd data-dir files. -- Add [`etcd --enable-log-rotation`](https://github.com/etcd-io/etcd/pull/12774) boolean flag which enables log rotation if true. -- Add [`etcd --log-rotation-config-json`](https://github.com/etcd-io/etcd/pull/12774) flag which allows passthrough of JSON config to configure log rotation for a file output target. -- Add experimental distributed tracing boolean flag [`--experimental-enable-distributed-tracing`](https://github.com/etcd-io/etcd/pull/12919) which enables tracing. -- Add [`etcd --experimental-distributed-tracing-address`](https://github.com/etcd-io/etcd/pull/12919) string flag which allows configuring the OpenTelemetry collector address. -- Add [`etcd --experimental-distributed-tracing-service-name`](https://github.com/etcd-io/etcd/pull/12919) string flag which allows changing the default "etcd" service name. -- Add [`etcd --experimental-distributed-tracing-instance-id`](https://github.com/etcd-io/etcd/pull/12919) string flag which configures an instance ID, which must be unique per etcd instance. - -### Package `runtime` - -- Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). - -### Package `embed` - -- Remove [`embed.Config.Debug`](https://github.com/etcd-io/etcd/pull/10947). - - Use `embed.Config.LogLevel` instead. -- Add [`embed.Config.ZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/11147) to allow creating a custom zap logger. -- Replace [global `*zap.Logger` with etcd server logger object](https://github.com/etcd-io/etcd/pull/12212). -- Add [`embed.Config.EnableLogRotation`](https://github.com/etcd-io/etcd/pull/12774) which enables log rotation if true. -- Add [`embed.Config.LogRotationConfigJSON`](https://github.com/etcd-io/etcd/pull/12774) to allow passthrough of JSON config to configure log rotation for a file output target. -- Add [`embed.Config.ExperimentalEnableDistributedTracing`](https://github.com/etcd-io/etcd/pull/12919) which enables experimental distributed tracing if true. -- Add [`embed.Config.ExperimentalDistributedTracingAddress`](https://github.com/etcd-io/etcd/pull/12919) which allows overriding default collector address. -- Add [`embed.Config.ExperimentalDistributedTracingServiceName`](https://github.com/etcd-io/etcd/pull/12919) which allows overriding default "etcd" service name. -- Add [`embed.Config.ExperimentalDistributedTracingServiceInstanceID`](https://github.com/etcd-io/etcd/pull/12919) which allows configuring an instance ID, which must be uniquer per etcd instance. - -### Package `clientv3` - -- Remove [excessive watch cancel logging messages](https://github.com/etcd-io/etcd/pull/12187). - - See [kubernetes/kubernetes#93450](https://github.com/kubernetes/kubernetes/issues/93450). -- Add [`TryLock`](https://github.com/etcd-io/etcd/pull/11104) method to `clientv3/concurrency/Mutex`. A non-blocking method on `Mutex` which does not wait to get lock on the Mutex, returns immediately if Mutex is locked by another session. -- Fix [client balancer failover against multiple endpoints](https://github.com/etcd-io/etcd/pull/11184). - - Fix [`"kube-apiserver: failover on multi-member etcd cluster fails certificate check on DNS mismatch"`](https://github.com/kubernetes/kubernetes/issues/83028). -- Fix [IPv6 endpoint parsing in client](https://github.com/etcd-io/etcd/pull/11211). - - Fix ["1.16: etcd client does not parse IPv6 addresses correctly when members are joining" (kubernetes#83550)](https://github.com/kubernetes/kubernetes/issues/83550). -- Fix [errors caused by grpc changing balancer/resolver API](https://github.com/etcd-io/etcd/pull/11564). This change is compatible with grpc >= [v1.26.0](https://github.com/grpc/grpc-go/releases/tag/v1.26.0), but is not compatible with < v1.26.0 version. -- Use [ServerName as the authority](https://github.com/etcd-io/etcd/pull/11574) after bumping to grpc v1.26.0. Remove workaround in [#11184](https://github.com/etcd-io/etcd/pull/11184). -- Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. -- Fix [watch leak caused by lazy cancellation](https://github.com/etcd-io/etcd/pull/11850). When clients cancel their watches, a cancel request will now be immediately sent to the server instead of waiting for the next watch event. -- Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). -- Fix [auth token invalid after watch reconnects](https://github.com/etcd-io/etcd/pull/12264). Get AuthToken automatically when clientConn is ready. -- Improve [clientv3:get AuthToken gracefully without extra connection](https://github.com/etcd-io/etcd/pull/12165). -- Changed [clientv3 dialing code](https://github.com/etcd-io/etcd/pull/12671) to use grpc resolver API instead of custom balancer. - - Endpoints self identify now as `etcd-endpoints://{id}/#initially={list of endpoints}` e.g. `etcd-endpoints://0xc0009d8540/#initially=[localhost:2079]` -- Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). - -### Package `lease` - -- Fix [memory leak in follower nodes](https://github.com/etcd-io/etcd/pull/11731). - - https://github.com/etcd-io/etcd/issues/11495 - - https://github.com/etcd-io/etcd/issues/11730 -- Make sure [grant/revoke won't be applied repeatedly after restarting etcd](https://github.com/etcd-io/etcd/pull/11935). - -### Package `wal` - -- Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). -- Handle [out-of-range slice bound in `ReadAll` and entry limit in `decodeRecord`](https://github.com/etcd-io/etcd/pull/11793). - -### etcdctl v3 - -- Fix `etcdctl member add` command to prevent potential timeout. ([PR#11194](https://github.com/etcd-io/etcd/pull/11194) and [PR#11638](https://github.com/etcd-io/etcd/pull/11638)) -- Add [`etcdctl watch --progress-notify`](https://github.com/etcd-io/etcd/pull/11462) flag. -- Add [`etcdctl auth status`](https://github.com/etcd-io/etcd/pull/11536) command to check if authentication is enabled -- Add [`etcdctl get --count-only`](https://github.com/etcd-io/etcd/pull/11743) flag for output type `fields`. -- Add [`etcdctl member list -w=json --hex`](https://github.com/etcd-io/etcd/pull/11812) flag to print memberListResponse in hex format json. -- Changed [`etcdctl lock exec-command`](https://github.com/etcd-io/etcd/pull/12829) to return exit code of exec-command. -- [New tool: `etcdutl`](https://github.com/etcd-io/etcd/pull/12971) incorporated functionality of: `etcdctl snapshot status|restore`, `etcdctl backup`, `etcdctl defrag --data-dir ...`. -- [ETCDCTL_API=2 `etcdctl migrate`](https://github.com/etcd-io/etcd/pull/12971) has been decomissioned. Use etcd <=v3.4 to restore v2 storage. - -### gRPC gateway - -- [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) only supports [`/v3`](TODO) endpoint. - - Deprecated [`/v3beta`](https://github.com/etcd-io/etcd/pull/9298). - - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` does work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - -### gRPC Proxy - -- Fix [`panic on error`](https://github.com/etcd-io/etcd/pull/11694) for metrics handler. -- Add [gRPC keepalive related flags](https://github.com/etcd-io/etcd/pull/11711) `grpc-keepalive-min-time`, `grpc-keepalive-interval` and `grpc-keepalive-timeout`. -- [Fix grpc watch proxy hangs when failed to cancel a watcher](https://github.com/etcd-io/etcd/pull/12030) . -- Add [metrics handler for grpcproxy self](https://github.com/etcd-io/etcd/pull/12107). -- Add [health handler for grpcproxy self](https://github.com/etcd-io/etcd/pull/12114). - -### Auth - -- Fix [NoPassword check when adding user through GRPC gateway](https://github.com/etcd-io/etcd/pull/11418) ([issue#11414](https://github.com/etcd-io/etcd/issues/11414)) -- Fix bug where [some auth related messages are logged at wrong level](https://github.com/etcd-io/etcd/pull/11586) -- [Fix a data corruption bug by saving consistent index](https://github.com/etcd-io/etcd/pull/11652). -- [Improve checkPassword performance](https://github.com/etcd-io/etcd/pull/11735). -- [Add authRevision field in AuthStatus](https://github.com/etcd-io/etcd/pull/11659). - -### API - -- Add [`/v3/auth/status`](https://github.com/etcd-io/etcd/pull/11536) endpoint to check if authentication is enabled -- [Add `Linearizable` field to `etcdserverpb.MemberListRequest`](https://github.com/etcd-io/etcd/pull/11639). -- [Learner support Snapshot RPC](https://github.com/etcd-io/etcd/pull/12890/). - -### Package `netutil` - -- Remove [`netutil.DropPort/RecoverPort/SetLatency/RemoveLatency`](https://github.com/etcd-io/etcd/pull/12491). - - These are not used anymore. They were only used for older versions of functional testing. - - Removed to adhere to best security practices, minimize arbitrary shell invocation. - -### `tools/etcd-dump-metrics` - -- Implement [input validation to prevent arbitrary shell invocation](https://github.com/etcd-io/etcd/pull/12491). - -### Dependency - -- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0) to [**`v1.37.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.37.0). -- Upgrade [`go.uber.org/zap`](https://github.com/uber-go/zap/releases) from [**`v1.14.1`**](https://github.com/uber-go/zap/releases/tag/v1.14.1) to [**`v1.16.0`**](https://github.com/uber-go/zap/releases/tag/v1.16.0). - -### Platforms - -- etcd now [officially supports `arm64`](https://github.com/etcd-io/etcd/pull/12929). - - See https://github.com/etcd-io/etcd/pull/12928 for adding automated tests with `arm64` EC2 instances (Graviton 2). - - See https://github.com/etcd-io/website/pull/273 for new platform support tier policies. - -### Release - -- Add s390x build support ([PR#11548](https://github.com/etcd-io/etcd/pull/11548) and [PR#11358](https://github.com/etcd-io/etcd/pull/11358)) - -### Go - -- Require [*Go 1.16+*](https://github.com/etcd-io/etcd/pull/11110). -- Compile with [*Go 1.16+*](https://golang.org/doc/devel/release.html#go1.16) -- etcd uses [go modules](https://github.com/etcd-io/etcd/pull/12279) (instead of vendor dir) to track dependencies. - -### Project Governance - -- The etcd team has added, a well defined and openly discussed, project [governance](https://github.com/etcd-io/etcd/pull/11175). - - -
- diff --git a/CHANGELOG-4.0.md b/CHANGELOG-4.0.md deleted file mode 100644 index 6b15f723719a..000000000000 --- a/CHANGELOG-4.0.md +++ /dev/null @@ -1,53 +0,0 @@ - - -Previous change logs can be found at [CHANGELOG-3.x](https://github.com/etcd-io/etcd/blob/main/CHANGELOG-3.x.md). - - -The minimum recommended etcd versions to run in **production** are 3.2.28+, 3.3.18+, and 3.4.2+. - - -
- - -## v4.0.0 (TBD) - -See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v4.0.0) and [v4.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_4_0/) for any breaking changes. - -**Again, before running upgrades from any previous release, please make sure to read change logs below and [v4.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_4_0/).** - -### Breaking Changes - -- [Secure etcd by default](https://github.com/etcd-io/etcd/issues/9475)? -- Change `/health` endpoint output. - - Previously, `{"health":"true"}`. - - Now, `{"health":true}`. - - Breaks [Kubernetes `kubectl get componentstatuses` command](https://github.com/kubernetes/kubernetes/issues/58240). -- Deprecate [`etcd --proxy*`](TODO) flags; **no more v2 proxy**. -- Deprecate [v2 storage backend](https://github.com/etcd-io/etcd/issues/9232); **no more v2 store**. - - v2 API is still supported via [v2 emulation](TODO). -- Deprecate [`etcdctl backup`](TODO) command. -- `clientv3.Client.KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error)` is now [`clientv4.Client.KeepAlive(ctx context.Context, id LeaseID) <-chan *LeaseKeepAliveResponse`](TODO). - - Similar to `Watch`, [`KeepAlive` does not return errors](https://github.com/etcd-io/etcd/issues/7488). - - If there's an unknown server error, kill all open channels and create a new stream on the next `KeepAlive` call. -- Rename `github.com/coreos/client` to `github.com/coreos/clientv2`. -- [`etcd --experimental-initial-corrupt-check`](TODO) has been deprecated. - - Use [`etcd --initial-corrupt-check`](TODO) instead. -- [`etcd --experimental-corrupt-check-time`](TODO) has been deprecated. - - Use [`etcd --corrupt-check-time`](TODO) instead. -- Enable TLS 1.13, deprecate TLS cipher suites. - -### etcd server - -- [`etcd --initial-corrupt-check`](TODO) flag is now stable (`etcd --experimental-initial-corrupt-check` has been deprecated). - - `etcd --initial-corrupt-check=true` by default, to check cluster database hashes before serving client/peer traffic. -- [`etcd --corrupt-check-time`](TODO) flag is now stable (`etcd --experimental-corrupt-check-time` has been deprecated). - - `etcd --corrupt-check-time=12h` by default, to check cluster database hashes for every 12-hour. -- Enable TLS 1.13, deprecate TLS cipher suites. - -### Go - -- Require [*Go 2*](https://blog.golang.org/go2draft). - - -
- diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b1d78dcfa91..3781d09e6e36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ This is a rough outline of what a contributor's workflow looks like: - Make sure commit messages are in the proper format (see below). - Push changes in a topic branch to a personal fork of the repository. - Submit a pull request to etcd-io/etcd. -- The PR must receive a LGTM from two maintainers found in the MAINTAINERS file. +- The PR must receive a LGTM from two maintainers found in the OWNERS file. Thanks for contributing! diff --git a/Dockerfile-release.amd64 b/Dockerfile-release.amd64 index 9bd425887cd2..934cc13544cb 100644 --- a/Dockerfile-release.amd64 +++ b/Dockerfile-release.amd64 @@ -1,16 +1,11 @@ -FROM k8s.gcr.io/build-image/debian-base:buster-v1.4.0 +FROM --platform=linux/amd64 gcr.io/distroless/static-debian11@sha256:1dbe426d60caed5d19597532a2d74c8056cd7b1674042b88f7328690b5ead8ed ADD etcd /usr/local/bin/ ADD etcdctl /usr/local/bin/ ADD etcdutl /usr/local/bin/ -RUN mkdir -p /var/etcd/ -RUN mkdir -p /var/lib/etcd/ -# Alpine Linux doesn't use pam, which means that there is no /etc/nsswitch.conf, -# but Golang relies on /etc/nsswitch.conf to check the order of DNS resolving -# (see https://github.com/golang/go/commit/9dee7771f561cf6aee081c0af6658cc81fac3918) -# To fix this we just create /etc/nsswitch.conf and add the following line: -RUN echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf +WORKDIR /var/etcd/ +WORKDIR /var/lib/etcd/ EXPOSE 2379 2380 diff --git a/Dockerfile-release.arm64 b/Dockerfile-release.arm64 index d04d79041a8c..a82bfd482379 100644 --- a/Dockerfile-release.arm64 +++ b/Dockerfile-release.arm64 @@ -1,11 +1,11 @@ -FROM k8s.gcr.io/build-image/debian-base-arm64:buster-v1.4.0 +FROM --platform=linux/arm64 gcr.io/distroless/static-debian11@sha256:1dbe426d60caed5d19597532a2d74c8056cd7b1674042b88f7328690b5ead8ed ADD etcd /usr/local/bin/ ADD etcdctl /usr/local/bin/ ADD etcdutl /usr/local/bin/ -ADD var/etcd /var/etcd -ADD var/lib/etcd /var/lib/etcd -ENV ETCD_UNSUPPORTED_ARCH=arm64 + +WORKDIR /var/etcd/ +WORKDIR /var/lib/etcd/ EXPOSE 2379 2380 diff --git a/Dockerfile-release.ppc64le b/Dockerfile-release.ppc64le index 51adb7ae3afd..7197afda7095 100644 --- a/Dockerfile-release.ppc64le +++ b/Dockerfile-release.ppc64le @@ -1,10 +1,11 @@ -FROM k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.4.0 +FROM --platform=linux/ppc64le gcr.io/distroless/static-debian11@sha256:1dbe426d60caed5d19597532a2d74c8056cd7b1674042b88f7328690b5ead8ed ADD etcd /usr/local/bin/ ADD etcdctl /usr/local/bin/ ADD etcdutl /usr/local/bin/ -ADD var/etcd /var/etcd -ADD var/lib/etcd /var/lib/etcd + +WORKDIR /var/etcd/ +WORKDIR /var/lib/etcd/ EXPOSE 2379 2380 diff --git a/Dockerfile-release.s390x b/Dockerfile-release.s390x index a96d45534c08..bfe87c71f6b5 100644 --- a/Dockerfile-release.s390x +++ b/Dockerfile-release.s390x @@ -1,10 +1,11 @@ -FROM k8s.gcr.io/build-image/debian-base-s390x:buster-v1.4.0 +FROM --platform=linux/s390x gcr.io/distroless/static-debian11@sha256:1dbe426d60caed5d19597532a2d74c8056cd7b1674042b88f7328690b5ead8ed ADD etcd /usr/local/bin/ ADD etcdctl /usr/local/bin/ ADD etcdutl /usr/local/bin/ -ADD var/etcd /var/etcd -ADD var/lib/etcd /var/lib/etcd + +WORKDIR /var/etcd/ +WORKDIR /var/lib/etcd/ EXPOSE 2379 2380 diff --git a/Dockerfile.art b/Dockerfile.art new file mode 100644 index 000000000000..e09885d97934 --- /dev/null +++ b/Dockerfile.art @@ -0,0 +1,26 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.21 AS builder + +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN ls -lR $REMOTE_SOURCES_DIR +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env && GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd + +# stage 2 (note: any changes should reflect in Dockerfile.rhel) +FROM registry.ci.openshift.org/ocp/4.21:base-rhel9 + +ENTRYPOINT ["/usr/bin/etcd"] + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcd /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdctl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdutl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/discover-etcd-initial-cluster /usr/bin/ + +LABEL io.k8s.display-name="etcd server" \ + io.k8s.description="etcd is a distributed key-value store which stores the persistent master state for Kubernetes and OpenShift." \ + maintainer="Sam Batschelet " diff --git a/Dockerfile.art-cachi2 b/Dockerfile.art-cachi2 new file mode 100644 index 000000000000..62749eaed06c --- /dev/null +++ b/Dockerfile.art-cachi2 @@ -0,0 +1,21 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +# stage 2 (note: any changes should reflect in Dockerfile.rhel) +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +ENTRYPOINT ["/usr/bin/etcd"] + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcd /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdctl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdutl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/discover-etcd-initial-cluster /usr/bin/ + +LABEL io.k8s.display-name="etcd server" \ + io.k8s.description="etcd is a distributed key-value store which stores the persistent master state for Kubernetes and OpenShift." \ + maintainer="Sam Batschelet " diff --git a/Dockerfile.installer b/Dockerfile.installer new file mode 100644 index 000000000000..4a6b896f5111 --- /dev/null +++ b/Dockerfile.installer @@ -0,0 +1,48 @@ +# This Dockerfile builds an image containing Mac and Linux ARM64/AMD64 versions of the etcd. +# The resulting image is used to build the statically-linked openshift-installer binary. + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS macbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS macarmbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS linuxbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS linuxarmbuilder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder +ENV GO_COMPLIANCE_EXCLUDE=".*" +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN CGO_ENABLED=0 GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh +RUN mkdir -p /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH) && \ + mv bin/etcd /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH)/ + +# stage 2 +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=macbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/amd64/etcd +COPY --from=macarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/arm64/etcd +COPY --from=linuxbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/amd64/etcd +COPY --from=linuxarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/arm64/etcd +COPY --from=builder /usr/share/openshift/ /usr/share/openshift/ + +# This image is not an operator, it is only used as part of the build pipeline +LABEL io.openshift.release.operator=false diff --git a/Dockerfile.installer.art b/Dockerfile.installer.art new file mode 100644 index 000000000000..a9c282d7b696 --- /dev/null +++ b/Dockerfile.installer.art @@ -0,0 +1,87 @@ +# This Dockerfile builds an image containing Mac and Linux ARM64/AMD64 versions of the etcd. +# The resulting image is used to build the statically-linked openshift-installer binary. + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS macbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS macarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS linuxbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS linuxarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder +ENV GO_COMPLIANCE_EXCLUDE=".*" +COPY $REMOTE_SOURCES $REMOTE_SOURCES_DIR +WORKDIR $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app +RUN cat $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env +RUN mkdir -p /go/src/go.etcd.io/ +RUN ln -s $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/app /go/src/go.etcd.io/etcd +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN source $REMOTE_SOURCES_DIR/cachito-gomod-with-deps/cachito.env \ + && export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 ./build.sh +RUN mkdir -p /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH) && \ + mv bin/etcd /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH)/ + +# stage 2 +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=macbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/amd64/etcd +COPY --from=macarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/arm64/etcd +COPY --from=linuxbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/amd64/etcd +COPY --from=linuxarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/arm64/etcd +COPY --from=builder /usr/share/openshift/ /usr/share/openshift/ + +# This image is not an operator, it is only used as part of the build pipeline +LABEL io.openshift.release.operator=false diff --git a/Dockerfile.installer.art-cachi2 b/Dockerfile.installer.art-cachi2 new file mode 100644 index 000000000000..805208575e32 --- /dev/null +++ b/Dockerfile.installer.art-cachi2 @@ -0,0 +1,65 @@ +# This Dockerfile builds an image containing Mac and Linux ARM64/AMD64 versions of the etcd. +# The resulting image is used to build the statically-linked openshift-installer binary. + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS macbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . + +RUN find $GOPATH +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS macarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . + +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS linuxbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS linuxarmbuilder + +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 ./build.sh + +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder +ENV GO_COMPLIANCE_EXCLUDE=".*" + +WORKDIR /go/src/go.etcd.io/etcd +COPY . . +RUN export GOFLAGS='-mod=readonly' && export GO_BUILD_FLAGS='-v' \ + && CGO_ENABLED=0 ./build.sh +RUN mkdir -p /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH) && \ + mv bin/etcd /usr/share/openshift/$(go env GOOS)/$(go env GOHOSTARCH)/ + +# stage 2 +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=macbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/amd64/etcd +COPY --from=macarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/darwin/arm64/etcd +COPY --from=linuxbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/amd64/etcd +COPY --from=linuxarmbuilder /go/src/go.etcd.io/etcd/bin/etcd /usr/share/openshift/linux/arm64/etcd +COPY --from=builder /usr/share/openshift/ /usr/share/openshift/ + +# This image is not an operator, it is only used as part of the build pipeline +LABEL io.openshift.release.operator=false diff --git a/Dockerfile.rhel b/Dockerfile.rhel new file mode 100644 index 000000000000..a5462e270324 --- /dev/null +++ b/Dockerfile.rhel @@ -0,0 +1,27 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.23-openshift-4.19 AS builder + +WORKDIR /go/src/go.etcd.io/etcd + + +COPY . . + +RUN GOFLAGS='-mod=readonly' GO_BUILD_FLAGS='-v' ./build.sh +RUN make tests-ext-build \ + && gzip etcd-tests-ext + +# stage 2 (note: any changes should reflect in Dockerfile.art) +FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 + +ENTRYPOINT ["/usr/bin/etcd"] + +RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/* + +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcd /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdctl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/etcdutl /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/etcd-tests-ext.gz /usr/bin/ +COPY --from=builder /go/src/go.etcd.io/etcd/bin/discover-etcd-initial-cluster /usr/bin/ + +LABEL io.k8s.display-name="etcd server" \ + io.k8s.description="etcd is a distributed key-value store which stores the persistent master state for Kubernetes and OpenShift." \ + maintainer="Sam Batschelet " diff --git a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json index b873790f3802..78795c40b0c4 100644 --- a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json @@ -1276,6 +1276,39 @@ } }, "/v3/maintenance/hash": { + "post": { + "tags": [ + "Maintenance" + ], + "summary": "Hash computes the hash of whole backend keyspace,\nincluding key, lease, and other buckets in storage.\nThis is designed for testing ONLY!\nDo not rely on this in production with ongoing transactions,\nsince Hash operation does not hold MVCC locks.\nUse \"HashKV\" API instead for \"key\" bucket consistency checks.", + "operationId": "Maintenance_Hash", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/etcdserverpbHashRequest" + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/etcdserverpbHashResponse" + } + }, + "default": { + "description": "An unexpected error response", + "schema": { + "$ref": "#/definitions/runtimeError" + } + } + } + } + }, + "/v3/maintenance/hashkv": { "post": { "tags": [ "Maintenance" diff --git a/GOVERNANCE.md b/GOVERNANCE.md index a0a6a290e567..dbd686e39527 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -14,7 +14,7 @@ please see [CONTRIBUTING](./CONTRIBUTING.md) guide. ## Maintainers -[Maintainers](./MAINTAINERS) are first and foremost contributors that have shown they +[Maintainers](./OWNERS) are first and foremost contributors that have shown they are committed to the long term success of a project. Maintainership is about building trust with the current maintainers of the project and being a person that they can depend on to make decisions in the best interest of the project in a consistent manner. @@ -48,12 +48,12 @@ of the maintainers and moved to emeritus status. Life priorities, interests, and passions can change. If a maintainer needs to step down, inform other maintainers about this intention, and if possible, help find someone to pick up the related work. At the very least, ensure the related work can be continued. -Afterward, create a pull request to remove yourself from the [MAINTAINERS](./MAINTAINERS) +Afterward, create a pull request to remove yourself from the [MAINTAINERS](./OWNERS) file. ## Reviewers -[Reviewers](./MAINTAINERS) are contributors who have demonstrated greater skill in +[Reviewers](./OWNERS) are contributors who have demonstrated greater skill in reviewing the code contribution from other contributors. Their LGTM counts towards merging a code change into the project. A reviewer is generally on the ladder towards maintainership. New reviewers must be nominated by an existing maintainer and must be diff --git a/MAINTAINERS b/MAINTAINERS deleted file mode 100644 index ecf320ad39ed..000000000000 --- a/MAINTAINERS +++ /dev/null @@ -1,24 +0,0 @@ -# The official list of maintainers and reviewers for the project maintenance. -# -# Refer to the GOVERNANCE.md for description of the roles. -# -# Names should be added to this file like so: -# Individual's name (@GITHUB_HANDLE) pkg:* -# Individual's name (@GITHUB_HANDLE) pkg:* -# -# Please keep the list sorted. - -# MAINTAINERS -Brandon Philips (@philips) pkg:* -Gyuho Lee (@gyuho) pkg:* -Hitoshi Mitake (@mitake) pkg:* -Jingyi Hu (@jingyih) pkg:* -Joe Betz (@jpbetz) pkg:* -Piotr Tabor (@ptabor) pkg:* -Sahdev Zala (@spzala) pkg:* -Sam Batschelet (@hexfusion) pkg:* -Wenjia Zhang (@wenjiaswe) pkg:* -Xiang Li (@xiang90) pkg:* - -Ben Darnell (@bdarnell) pkg:go.etcd.io/etcd/raft -Tobias Grieger (@tbg) pkg:go.etcd.io/etcd/raft diff --git a/Makefile b/Makefile index bfd6741c6017..ae97aaa4dd41 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ clean: rm -rf ./release rm -rf ./coverage/*.err ./coverage/*.out rm -rf ./tests/e2e/default.proxy + rm -rf ./bin/shellcheck* find ./ -name "127.0.0.1:*" -o -name "localhost:*" -o -name "*.log" -o -name "agent-*" -o -name "*.coverprofile" -o -name "testname-proxy-*" | $(XARGS) docker-clean: @@ -55,7 +56,7 @@ docker-remove: -GO_VERSION ?= 1.16.3 +GO_VERSION ?= $(shell cat .go-version) ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound") TEST_SUFFIX = $(shell date +%s | base64 | head -c 15) @@ -155,13 +156,33 @@ test: test-smoke: $(info log-file: test-$(TEST_SUFFIX).log) - PASSES="fmt build unit" ./test.sh 2<&1 | tee test-$(TEST_SUFFIX).log + PASSES="fmt bom dep build unit" ./test.sh 2<&1 | tee test-$(TEST_SUFFIX).log + ! grep "FAIL:" test-$(TEST_SUFFIX).log test-full: $(info log-file: test-$(TEST_SUFFIX).log) PASSES="fmt build release unit integration functional e2e grpcproxy" ./test.sh 2<&1 | tee test-$(TEST_SUFFIX).log -docker-test: +.PHONY: test-unit +test-unit: + PASSES="unit" ./test.sh $(GO_TEST_FLAGS) + +.PHONY: test-integration +test-integration: + PASSES="integration" ./test.sh $(GO_TEST_FLAGS) + +.PHONY: test-e2e +test-e2e: + PASSES="build e2e" ./test.sh $(GO_TEST_FLAGS) + +.PHONY: test-e2e-release +test-e2e-release: + PASSES="build release e2e" ./test.sh $(GO_TEST_FLAGS) + +ensure-docker-test-image-exists: + make pull-docker-test || ( echo "WARNING: Container Image not found in registry, building locally"; make build-docker-test ) + +docker-test: ensure-docker-test-image-exists $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) $(info TEST_OPTS: $(TEST_OPTS)) @@ -546,3 +567,82 @@ pull-docker-functional: $(info GO_VERSION: $(GO_VERSION)) $(info ETCD_VERSION: $(ETCD_VERSION)) docker pull gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) + +# Failpoints +GOFAIL_VERSION = $(shell cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail) +.PHONY: install-gofail +install-gofail: + go install go.etcd.io/gofail@${GOFAIL_VERSION} + +.PHONY: gofail-enable +gofail-enable: install-gofail + PASSES="toggle_failpoints" FAILPOINTS=true ./test.sh + +.PHONY: gofail-disable +gofail-disable: install-gofail + PASSES="toggle_failpoints" ./test.sh + +.PHONY: verify +verify: verify-go-versions verify-dep + +.PHONY: verify-go-versions +verify-go-versions: + ./scripts/verify_go_versions.sh + +.PHONY: verify-dep +verify-dep: + PASSES="dep" ./test.sh 2<&1 + +.PHONY: fix +fix: sync-toolchain-directive + ./scripts/fix.sh + +.PHONY: sync-toolchain-directive +sync-toolchain-directive: + ./scripts/sync_go_toolchain_directive.sh + + +# OpenShift Tests Extension variables +TESTS_EXT_BINARY ?= etcd-tests-ext +TESTS_EXT_PACKAGE ?= ./cmd/etcd-tests-ext +TESTS_EXT_LDFLAGS ?= -X 'main.CommitFromGit=$(shell git rev-parse --short HEAD)' \ + -X 'main.BuildDate=$(shell date -u +%Y-%m-%dT%H:%M:%SZ)' \ + -X 'main.GitTreeState=$(shell if git diff-index --quiet HEAD --; then echo clean; else echo dirty; fi)' + +# Build the openshift-tests-extension binary +.PHONY: tests-ext-build +tests-ext-build: + GOOS=$(GOOS) GOARCH=$(GOARCH) GO_COMPLIANCE_POLICY=exempt_all CGO_ENABLED=0 \ + go build -o $(TESTS_EXT_BINARY) -ldflags "$(TESTS_EXT_LDFLAGS)" $(TESTS_EXT_PACKAGE) + +# Update test metadata +.PHONY: tests-ext-update +tests-ext-update: + ./$(TESTS_EXT_BINARY) update + +# Clean tests extension artifacts +.PHONY: tests-ext-clean +tests-ext-clean: + rm -f $(TESTS_EXT_BINARY) $(TESTS_EXT_BINARY).gz + +# Run tests extension help +.PHONY: tests-ext-help +tests-ext-help: + ./$(TESTS_EXT_BINARY) --help + + +# Run sanity test +.PHONY: tests-ext-sanity +tests-ext-sanity: + ./$(TESTS_EXT_BINARY) run-suite "openshift/etcd/conformance/parallel" + +# List available tests +.PHONY: tests-ext-list +tests-ext-list: + ./$(TESTS_EXT_BINARY) list tests + +# Show extension info +.PHONY: tests-ext-info +tests-ext-info: + ./$(TESTS_EXT_BINARY) info + diff --git a/OWNERS b/OWNERS new file mode 100644 index 000000000000..4a00b7e4aa0c --- /dev/null +++ b/OWNERS @@ -0,0 +1,13 @@ +approvers: +- deads2k +- hasbro17 +- dusk125 +- Elbehery +- tjungblu +reviewers: +- deads2k +- dusk125 +- hasbro17 +- Elbehery +- tjungblu +component: "Etcd" diff --git a/README.md b/README.md index fc438ae732e8..212c3b0ed37f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/etcd?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/etcd) [![Coverage](https://codecov.io/gh/etcd-io/etcd/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/etcd) [![Tests](https://github.com/etcd-io/etcd/actions/workflows/tests.yaml/badge.svg)](https://github.com/etcd-io/etcd/actions/workflows/tests.yaml) -[![asset-transparency](https://github.com/etcd-io/etcd/actions/workflows/asset-transparency.yaml/badge.svg)](https://github.com/etcd-io/etcd/actions/workflows/asset-transparency.yaml) [![codeql-analysis](https://github.com/etcd-io/etcd/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/etcd-io/etcd/actions/workflows/codeql-analysis.yml) [![self-hosted-linux-arm64-graviton2-tests](https://github.com/etcd-io/etcd/actions/workflows/self-hosted-linux-arm64-graviton2-tests.yml/badge.svg)](https://github.com/etcd-io/etcd/actions/workflows/self-hosted-linux-arm64-graviton2-tests.yml) [![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://etcd.io/docs) diff --git a/REBASE.openshift.md b/REBASE.openshift.md new file mode 100644 index 000000000000..51b27ae23a64 --- /dev/null +++ b/REBASE.openshift.md @@ -0,0 +1,87 @@ +# Maintaining openshift/etcd + +OpenShift is based on upstream etcd. With every release of etcd that is +intended to be shipped as OCP, it is necessary to incorporate the upstream changes +while ensuring that our downstream customizations are maintained. + +## Maintaining this document + +An openshift/etcd rebase is a complex process involving many manual and +potentially error-prone steps. If, while performing a rebase, you find areas where +the documented procedure is unclear or missing detail, please update this document +and include the change in the rebase PR. This will ensure that the instructions are +as comprehensive and accurate as possible for the person performing the next +rebase. + +## Getting started + +Before incorporating upstream changes you may want to: + +- Read this document +- Find the best tool for resolving merge conflicts +- Use diff3 conflict resolution strategy + (https://blog.nilbus.com/take-the-pain-out-of-git-conflict-resolution-use-diff3/) +- Teach Git to remember how you’ve resolved a conflict so that the next time it can + resolve it automatically (https://git-scm.com/book/en/v2/Git-Tools-Rerere) + +## Preparing the local repo clone + +Clone from a personal fork of etcd via a pushable (ssh) url: + +``` +git clone git@github.com:/etcd +``` + +## Updating with `rebase.sh` + +To finally rebase, a script that will merge and rebase along the happy path without automatic conflict resolution and at the end will create a PR for you. + +Here are the steps: +1. Create a new OCPBUGS JIRA ticket with the respective OpenShift version to rebase. Please include the change logs in the ticket description. You can clone a previous rebase we did in [OCPBUGS-947](https://issues.redhat.com/browse/OCPBUGS-947) and adjust. +2. It's best to start off with a fresh fork of [openshift/etcd](https://github.com/openshift/etcd/). Stay on the master branch. +3. This script requires `jq`, `git`, `podman` and `bash`. `gh` is optional. +4. In the root dir of that fork run: +``` +openshift-hack/rebase.sh --etcd-tag=v3.5.4 --openshift-release=openshift-4.12 --jira-id=666 +``` + +where `etcd-tag` is the [etcd-io/etcd](https://github.com/etcd-io/etcd/) release tag, the `openshift-release` +is the OpenShift release branch in [openshift/etcd](https://github.com/openshift/etcd/) and the `jira-id` is the +number of the OCPBUGS ticket created in step (1). + +5. In case of conflicts, it will ask you to step into another shell to resolve those. The script will continue by committing the resolution with `UPSTREAM: `. +6. At the end, there will be a "rebase-$VERSION" branch pushed to your fork. +7. If you have `gh` installed and are logged in, it will attempt to create a PR for you by opening a web browser. + +## Building and testing + +- Build the code with `make` +- Test the code with `make test` + +## Payload testing + +After all the above are green and your PR pre-submits are too, you can start with payload testing. This is to ensure the nightly jobs won't break on etcd after the merge, they also test all of OpenShift (including upgrades) well enough. + +You should run those two: + +> /payload 4.x nightly informing +> +> /payload 4.x nightly blocking + +Replace 4.x with the respective OpenShift release you're merging against. + +It pays off to inspect some Prometheus metrics (CPU, memory, disk usage) with an upgrade job through PromeCIus. This is to ensure we don't increase resource usage inadvertently. + +## Testing with ClusterBot + +Sometimes it's easier to debug an issue using cluster bot. Here you can simply run the given OpenShift release using your rebase PR: + +> launch openshift/etcd#155 + +This is particularly helpful when you want to test specific providers, for example Bare Metal or VSphere, or just other variants like SNO. + +## Performance Testing + +We currently do not do performance testing after an etcd rebase in OpenShift, however the upstream community and [SIG Scalability in k/k does](https://github.com/etcd-io/etcd/issues/14138#issuecomment-1247665949). + +The OpenShift scalability team also regularly runs performance tests with upcoming 4.y.0 releases. diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index d9898166c32e..000000000000 --- a/ROADMAP.md +++ /dev/null @@ -1,21 +0,0 @@ -# etcd roadmap - -**work in progress** - -This document defines a high level roadmap for etcd development. - -The dates below should not be considered authoritative, but rather indicative of the projected timeline of the project. The [milestones defined in GitHub](https://github.com/etcd-io/etcd/milestones) represent the most up-to-date and issue-for-issue plans. - -etcd 3.3 is our current stable branch. The roadmap below outlines new features that will be added to etcd, and while subject to change, define what future stable will look like. - -### etcd 3.4 (2019) - -- Stabilization of 3.3 experimental features -- Support/document downgrade -- Snapshot restore as Go library -- Improved client balancer with new gRPC balancer interface -- Improve single-client put performance -- Improve large response handling -- Improve test coverage -- Decrease test runtime -- Migrate to Go module for dependency management diff --git a/api/etcdserverpb/gw/rpc.pb.gw.go b/api/etcdserverpb/gw/rpc.pb.gw.go index 2fca126af851..62aba97cdb82 100644 --- a/api/etcdserverpb/gw/rpc.pb.gw.go +++ b/api/etcdserverpb/gw/rpc.pb.gw.go @@ -3290,7 +3290,7 @@ var ( pattern_Maintenance_Hash_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "hash"}, "", runtime.AssumeColonVerbOpt(true))) - pattern_Maintenance_HashKV_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "hash"}, "", runtime.AssumeColonVerbOpt(true))) + pattern_Maintenance_HashKV_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "hashkv"}, "", runtime.AssumeColonVerbOpt(true))) pattern_Maintenance_Snapshot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "snapshot"}, "", runtime.AssumeColonVerbOpt(true))) diff --git a/api/etcdserverpb/raft_internal_stringer_test.go b/api/etcdserverpb/raft_internal_stringer_test.go index 35be87824791..f6280e91351c 100644 --- a/api/etcdserverpb/raft_internal_stringer_test.go +++ b/api/etcdserverpb/raft_internal_stringer_test.go @@ -20,7 +20,7 @@ import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) -// TestInvalidGoYypeIntPanic tests conditions that caused +// TestInvalidGoTypeIntPanic tests conditions that caused // panic: invalid Go type int for field k8s_io.kubernetes.vendor.go_etcd_io.etcd.etcdserver.etcdserverpb.loggablePutRequest.value_size // See https://github.com/kubernetes/kubernetes/issues/91937 for more details func TestInvalidGoTypeIntPanic(t *testing.T) { diff --git a/api/etcdserverpb/rpc.pb.go b/api/etcdserverpb/rpc.pb.go index 34c1824426e5..8120907d91de 100644 --- a/api/etcdserverpb/rpc.pb.go +++ b/api/etcdserverpb/rpc.pb.go @@ -6154,264 +6154,264 @@ func init() { func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 4107 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0x5b, 0x73, 0x1b, 0xc9, - 0x75, 0xe6, 0x00, 0xc4, 0xed, 0xe0, 0x42, 0xb0, 0x79, 0x11, 0x84, 0x95, 0x28, 0x6e, 0x6b, 0xa5, - 0xe5, 0x4a, 0xbb, 0xc4, 0x9a, 0xb6, 0xb3, 0x55, 0x4a, 0xe2, 0x18, 0x22, 0xb1, 0x12, 0x97, 0x14, - 0xc9, 0x1d, 0x42, 0xda, 0x4b, 0xb9, 0xc2, 0x1a, 0x02, 0x2d, 0x72, 0x42, 0x60, 0x06, 0x9e, 0x19, - 0x40, 0xe4, 0xe6, 0xe2, 0x94, 0xcb, 0x71, 0x25, 0xaf, 0x76, 0x55, 0x2a, 0x79, 0x48, 0x5e, 0x52, - 0x29, 0x97, 0x1f, 0xfc, 0x9c, 0xbf, 0x90, 0xa7, 0x5c, 0x2a, 0x7f, 0x20, 0xb5, 0xf1, 0x4b, 0xf2, - 0x23, 0x52, 0xae, 0xbe, 0xcd, 0xf4, 0xdc, 0x40, 0xd9, 0xd8, 0xdd, 0x17, 0x11, 0x7d, 0xfa, 0xf4, - 0xf9, 0x4e, 0x9f, 0xee, 0x3e, 0xe7, 0xf4, 0xe9, 0x11, 0x94, 0x9c, 0x51, 0x6f, 0x73, 0xe4, 0xd8, - 0x9e, 0x8d, 0x2a, 0xc4, 0xeb, 0xf5, 0x5d, 0xe2, 0x4c, 0x88, 0x33, 0x3a, 0x6d, 0x2e, 0x9f, 0xd9, - 0x67, 0x36, 0xeb, 0x68, 0xd1, 0x5f, 0x9c, 0xa7, 0xd9, 0xa0, 0x3c, 0x2d, 0x63, 0x64, 0xb6, 0x86, - 0x93, 0x5e, 0x6f, 0x74, 0xda, 0xba, 0x98, 0x88, 0x9e, 0xa6, 0xdf, 0x63, 0x8c, 0xbd, 0xf3, 0xd1, - 0x29, 0xfb, 0x23, 0xfa, 0x6e, 0x9d, 0xd9, 0xf6, 0xd9, 0x80, 0xf0, 0x5e, 0xcb, 0xb2, 0x3d, 0xc3, - 0x33, 0x6d, 0xcb, 0xe5, 0xbd, 0xf8, 0xaf, 0x34, 0xa8, 0xe9, 0xc4, 0x1d, 0xd9, 0x96, 0x4b, 0x9e, - 0x12, 0xa3, 0x4f, 0x1c, 0x74, 0x1b, 0xa0, 0x37, 0x18, 0xbb, 0x1e, 0x71, 0x4e, 0xcc, 0x7e, 0x43, - 0x5b, 0xd7, 0x36, 0xe6, 0xf5, 0x92, 0xa0, 0xec, 0xf6, 0xd1, 0x1b, 0x50, 0x1a, 0x92, 0xe1, 0x29, - 0xef, 0xcd, 0xb0, 0xde, 0x22, 0x27, 0xec, 0xf6, 0x51, 0x13, 0x8a, 0x0e, 0x99, 0x98, 0xae, 0x69, - 0x5b, 0x8d, 0xec, 0xba, 0xb6, 0x91, 0xd5, 0xfd, 0x36, 0x1d, 0xe8, 0x18, 0x2f, 0xbd, 0x13, 0x8f, - 0x38, 0xc3, 0xc6, 0x3c, 0x1f, 0x48, 0x09, 0x5d, 0xe2, 0x0c, 0xf1, 0x4f, 0x72, 0x50, 0xd1, 0x0d, - 0xeb, 0x8c, 0xe8, 0xe4, 0x87, 0x63, 0xe2, 0x7a, 0xa8, 0x0e, 0xd9, 0x0b, 0x72, 0xc5, 0xe0, 0x2b, - 0x3a, 0xfd, 0xc9, 0xc7, 0x5b, 0x67, 0xe4, 0x84, 0x58, 0x1c, 0xb8, 0x42, 0xc7, 0x5b, 0x67, 0xa4, - 0x63, 0xf5, 0xd1, 0x32, 0xe4, 0x06, 0xe6, 0xd0, 0xf4, 0x04, 0x2a, 0x6f, 0x84, 0xd4, 0x99, 0x8f, - 0xa8, 0xb3, 0x0d, 0xe0, 0xda, 0x8e, 0x77, 0x62, 0x3b, 0x7d, 0xe2, 0x34, 0x72, 0xeb, 0xda, 0x46, - 0x6d, 0xeb, 0xad, 0x4d, 0x75, 0x19, 0x36, 0x55, 0x85, 0x36, 0x8f, 0x6d, 0xc7, 0x3b, 0xa4, 0xbc, - 0x7a, 0xc9, 0x95, 0x3f, 0xd1, 0x87, 0x50, 0x66, 0x42, 0x3c, 0xc3, 0x39, 0x23, 0x5e, 0x23, 0xcf, - 0xa4, 0xdc, 0xbb, 0x46, 0x4a, 0x97, 0x31, 0xeb, 0x0c, 0x9e, 0xff, 0x46, 0x18, 0x2a, 0x2e, 0x71, - 0x4c, 0x63, 0x60, 0x7e, 0x61, 0x9c, 0x0e, 0x48, 0xa3, 0xb0, 0xae, 0x6d, 0x14, 0xf5, 0x10, 0x8d, - 0xce, 0xff, 0x82, 0x5c, 0xb9, 0x27, 0xb6, 0x35, 0xb8, 0x6a, 0x14, 0x19, 0x43, 0x91, 0x12, 0x0e, - 0xad, 0xc1, 0x15, 0x5b, 0x34, 0x7b, 0x6c, 0x79, 0xbc, 0xb7, 0xc4, 0x7a, 0x4b, 0x8c, 0xc2, 0xba, - 0x37, 0xa0, 0x3e, 0x34, 0xad, 0x93, 0xa1, 0xdd, 0x3f, 0xf1, 0x0d, 0x02, 0xcc, 0x20, 0xb5, 0xa1, - 0x69, 0x3d, 0xb3, 0xfb, 0xba, 0x34, 0x0b, 0xe5, 0x34, 0x2e, 0xc3, 0x9c, 0x65, 0xc1, 0x69, 0x5c, - 0xaa, 0x9c, 0x9b, 0xb0, 0x44, 0x65, 0xf6, 0x1c, 0x62, 0x78, 0x24, 0x60, 0xae, 0x30, 0xe6, 0xc5, - 0xa1, 0x69, 0x6d, 0xb3, 0x9e, 0x10, 0xbf, 0x71, 0x19, 0xe3, 0xaf, 0x0a, 0x7e, 0xe3, 0x32, 0xcc, - 0x8f, 0x37, 0xa1, 0xe4, 0xdb, 0x1c, 0x15, 0x61, 0xfe, 0xe0, 0xf0, 0xa0, 0x53, 0x9f, 0x43, 0x00, - 0xf9, 0xf6, 0xf1, 0x76, 0xe7, 0x60, 0xa7, 0xae, 0xa1, 0x32, 0x14, 0x76, 0x3a, 0xbc, 0x91, 0xc1, - 0x8f, 0x01, 0x02, 0xeb, 0xa2, 0x02, 0x64, 0xf7, 0x3a, 0x9f, 0xd5, 0xe7, 0x28, 0xcf, 0x8b, 0x8e, - 0x7e, 0xbc, 0x7b, 0x78, 0x50, 0xd7, 0xe8, 0xe0, 0x6d, 0xbd, 0xd3, 0xee, 0x76, 0xea, 0x19, 0xca, - 0xf1, 0xec, 0x70, 0xa7, 0x9e, 0x45, 0x25, 0xc8, 0xbd, 0x68, 0xef, 0x3f, 0xef, 0xd4, 0xe7, 0xf1, - 0xcf, 0x35, 0xa8, 0x8a, 0xf5, 0xe2, 0x67, 0x02, 0x7d, 0x07, 0xf2, 0xe7, 0xec, 0x5c, 0xb0, 0xad, - 0x58, 0xde, 0xba, 0x15, 0x59, 0xdc, 0xd0, 0xd9, 0xd1, 0x05, 0x2f, 0xc2, 0x90, 0xbd, 0x98, 0xb8, - 0x8d, 0xcc, 0x7a, 0x76, 0xa3, 0xbc, 0x55, 0xdf, 0xe4, 0xe7, 0x75, 0x73, 0x8f, 0x5c, 0xbd, 0x30, - 0x06, 0x63, 0xa2, 0xd3, 0x4e, 0x84, 0x60, 0x7e, 0x68, 0x3b, 0x84, 0xed, 0xd8, 0xa2, 0xce, 0x7e, - 0xd3, 0x6d, 0xcc, 0x16, 0x4d, 0xec, 0x56, 0xde, 0xc0, 0xbf, 0xd4, 0x00, 0x8e, 0xc6, 0x5e, 0xfa, - 0xd1, 0x58, 0x86, 0xdc, 0x84, 0x0a, 0x16, 0xc7, 0x82, 0x37, 0xd8, 0x99, 0x20, 0x86, 0x4b, 0xfc, - 0x33, 0x41, 0x1b, 0xe8, 0x06, 0x14, 0x46, 0x0e, 0x99, 0x9c, 0x5c, 0x4c, 0x18, 0x48, 0x51, 0xcf, - 0xd3, 0xe6, 0xde, 0x04, 0xbd, 0x09, 0x15, 0xf3, 0xcc, 0xb2, 0x1d, 0x72, 0xc2, 0x65, 0xe5, 0x58, - 0x6f, 0x99, 0xd3, 0x98, 0xde, 0x0a, 0x0b, 0x17, 0x9c, 0x57, 0x59, 0xf6, 0x29, 0x09, 0x5b, 0x50, - 0x66, 0xaa, 0xce, 0x64, 0xbe, 0x77, 0x02, 0x1d, 0x33, 0x6c, 0x58, 0xdc, 0x84, 0x42, 0x6b, 0xfc, - 0x03, 0x40, 0x3b, 0x64, 0x40, 0x3c, 0x32, 0x8b, 0xf7, 0x50, 0x6c, 0x92, 0x55, 0x6d, 0x82, 0x7f, - 0xa6, 0xc1, 0x52, 0x48, 0xfc, 0x4c, 0xd3, 0x6a, 0x40, 0xa1, 0xcf, 0x84, 0x71, 0x0d, 0xb2, 0xba, - 0x6c, 0xa2, 0x87, 0x50, 0x14, 0x0a, 0xb8, 0x8d, 0x6c, 0xca, 0xa6, 0x29, 0x70, 0x9d, 0x5c, 0xfc, - 0xcb, 0x0c, 0x94, 0xc4, 0x44, 0x0f, 0x47, 0xa8, 0x0d, 0x55, 0x87, 0x37, 0x4e, 0xd8, 0x7c, 0x84, - 0x46, 0xcd, 0x74, 0x27, 0xf4, 0x74, 0x4e, 0xaf, 0x88, 0x21, 0x8c, 0x8c, 0x7e, 0x1f, 0xca, 0x52, - 0xc4, 0x68, 0xec, 0x09, 0x93, 0x37, 0xc2, 0x02, 0x82, 0xfd, 0xf7, 0x74, 0x4e, 0x07, 0xc1, 0x7e, - 0x34, 0xf6, 0x50, 0x17, 0x96, 0xe5, 0x60, 0x3e, 0x1b, 0xa1, 0x46, 0x96, 0x49, 0x59, 0x0f, 0x4b, - 0x89, 0x2f, 0xd5, 0xd3, 0x39, 0x1d, 0x89, 0xf1, 0x4a, 0xa7, 0xaa, 0x92, 0x77, 0xc9, 0x9d, 0x77, - 0x4c, 0xa5, 0xee, 0xa5, 0x15, 0x57, 0xa9, 0x7b, 0x69, 0x3d, 0x2e, 0x41, 0x41, 0xb4, 0xf0, 0xbf, - 0x64, 0x00, 0xe4, 0x6a, 0x1c, 0x8e, 0xd0, 0x0e, 0xd4, 0x1c, 0xd1, 0x0a, 0x59, 0xeb, 0x8d, 0x44, - 0x6b, 0x89, 0x45, 0x9c, 0xd3, 0xab, 0x72, 0x10, 0x57, 0xee, 0x7b, 0x50, 0xf1, 0xa5, 0x04, 0x06, - 0xbb, 0x99, 0x60, 0x30, 0x5f, 0x42, 0x59, 0x0e, 0xa0, 0x26, 0xfb, 0x04, 0x56, 0xfc, 0xf1, 0x09, - 0x36, 0x7b, 0x73, 0x8a, 0xcd, 0x7c, 0x81, 0x4b, 0x52, 0x82, 0x6a, 0x35, 0x55, 0xb1, 0xc0, 0x6c, - 0x37, 0x13, 0xcc, 0x16, 0x57, 0x8c, 0x1a, 0x0e, 0x68, 0xbc, 0xe4, 0x4d, 0xfc, 0xbf, 0x59, 0x28, - 0x6c, 0xdb, 0xc3, 0x91, 0xe1, 0xd0, 0xd5, 0xc8, 0x3b, 0xc4, 0x1d, 0x0f, 0x3c, 0x66, 0xae, 0xda, - 0xd6, 0xdd, 0xb0, 0x44, 0xc1, 0x26, 0xff, 0xea, 0x8c, 0x55, 0x17, 0x43, 0xe8, 0x60, 0x11, 0x1e, - 0x33, 0xaf, 0x31, 0x58, 0x04, 0x47, 0x31, 0x44, 0x1e, 0xe4, 0x6c, 0x70, 0x90, 0x9b, 0x50, 0x98, - 0x10, 0x27, 0x08, 0xe9, 0x4f, 0xe7, 0x74, 0x49, 0x40, 0xef, 0xc0, 0x42, 0x34, 0xbc, 0xe4, 0x04, - 0x4f, 0xad, 0x17, 0x8e, 0x46, 0x77, 0xa1, 0x12, 0x8a, 0x71, 0x79, 0xc1, 0x57, 0x1e, 0x2a, 0x21, - 0x6e, 0x55, 0xfa, 0x55, 0x1a, 0x8f, 0x2b, 0x4f, 0xe7, 0xa4, 0x67, 0x5d, 0x95, 0x9e, 0xb5, 0x28, - 0x46, 0x09, 0xdf, 0x1a, 0x72, 0x32, 0xdf, 0x0f, 0x3b, 0x19, 0xfc, 0x7d, 0xa8, 0x86, 0x0c, 0x44, - 0xe3, 0x4e, 0xe7, 0xe3, 0xe7, 0xed, 0x7d, 0x1e, 0xa4, 0x9e, 0xb0, 0xb8, 0xa4, 0xd7, 0x35, 0x1a, - 0xeb, 0xf6, 0x3b, 0xc7, 0xc7, 0xf5, 0x0c, 0xaa, 0x42, 0xe9, 0xe0, 0xb0, 0x7b, 0xc2, 0xb9, 0xb2, - 0xf8, 0x89, 0x2f, 0x41, 0x04, 0x39, 0x25, 0xb6, 0xcd, 0x29, 0xb1, 0x4d, 0x93, 0xb1, 0x2d, 0x13, - 0xc4, 0x36, 0x16, 0xe6, 0xf6, 0x3b, 0xed, 0xe3, 0x4e, 0x7d, 0xfe, 0x71, 0x0d, 0x2a, 0xdc, 0xbe, - 0x27, 0x63, 0x8b, 0x86, 0xda, 0x7f, 0xd2, 0x00, 0x82, 0xd3, 0x84, 0x5a, 0x50, 0xe8, 0x71, 0x9c, - 0x86, 0xc6, 0x9c, 0xd1, 0x4a, 0xe2, 0x92, 0xe9, 0x92, 0x0b, 0x7d, 0x0b, 0x0a, 0xee, 0xb8, 0xd7, - 0x23, 0xae, 0x0c, 0x79, 0x37, 0xa2, 0xfe, 0x50, 0x78, 0x2b, 0x5d, 0xf2, 0xd1, 0x21, 0x2f, 0x0d, - 0x73, 0x30, 0x66, 0x01, 0x70, 0xfa, 0x10, 0xc1, 0x87, 0xff, 0x5e, 0x83, 0xb2, 0xb2, 0x79, 0x7f, - 0x47, 0x27, 0x7c, 0x0b, 0x4a, 0x4c, 0x07, 0xd2, 0x17, 0x6e, 0xb8, 0xa8, 0x07, 0x04, 0xf4, 0x7b, - 0x50, 0x92, 0x27, 0x40, 0x7a, 0xe2, 0x46, 0xb2, 0xd8, 0xc3, 0x91, 0x1e, 0xb0, 0xe2, 0x3d, 0x58, - 0x64, 0x56, 0xe9, 0xd1, 0xe4, 0x5a, 0xda, 0x51, 0x4d, 0x3f, 0xb5, 0x48, 0xfa, 0xd9, 0x84, 0xe2, - 0xe8, 0xfc, 0xca, 0x35, 0x7b, 0xc6, 0x40, 0x68, 0xe1, 0xb7, 0xf1, 0x47, 0x80, 0x54, 0x61, 0xb3, - 0x4c, 0x17, 0x57, 0xa1, 0xfc, 0xd4, 0x70, 0xcf, 0x85, 0x4a, 0xf8, 0x21, 0x54, 0x69, 0x73, 0xef, - 0xc5, 0x6b, 0xe8, 0xc8, 0x2e, 0x07, 0x92, 0x7b, 0x26, 0x9b, 0x23, 0x98, 0x3f, 0x37, 0xdc, 0x73, - 0x36, 0xd1, 0xaa, 0xce, 0x7e, 0xa3, 0x77, 0xa0, 0xde, 0xe3, 0x93, 0x3c, 0x89, 0x5c, 0x19, 0x16, - 0x04, 0xdd, 0xcf, 0x04, 0x3f, 0x85, 0x0a, 0x9f, 0xc3, 0x57, 0xad, 0x04, 0x5e, 0x84, 0x85, 0x63, - 0xcb, 0x18, 0xb9, 0xe7, 0xb6, 0x8c, 0x6e, 0x74, 0xd2, 0xf5, 0x80, 0x36, 0x13, 0xe2, 0xdb, 0xb0, - 0xe0, 0x90, 0xa1, 0x61, 0x5a, 0xa6, 0x75, 0x76, 0x72, 0x7a, 0xe5, 0x11, 0x57, 0x5c, 0x98, 0x6a, - 0x3e, 0xf9, 0x31, 0xa5, 0x52, 0xd5, 0x4e, 0x07, 0xf6, 0xa9, 0x70, 0x73, 0xec, 0x37, 0xfe, 0x69, - 0x06, 0x2a, 0x9f, 0x18, 0x5e, 0x4f, 0x2e, 0x1d, 0xda, 0x85, 0x9a, 0xef, 0xdc, 0x18, 0x45, 0xe8, - 0x12, 0x09, 0xb1, 0x6c, 0x8c, 0x4c, 0xa5, 0x65, 0x74, 0xac, 0xf6, 0x54, 0x02, 0x13, 0x65, 0x58, - 0x3d, 0x32, 0xf0, 0x45, 0x65, 0xd2, 0x45, 0x31, 0x46, 0x55, 0x94, 0x4a, 0x40, 0x87, 0x50, 0x1f, - 0x39, 0xf6, 0x99, 0x43, 0x5c, 0xd7, 0x17, 0xc6, 0xc3, 0x18, 0x4e, 0x10, 0x76, 0x24, 0x58, 0x03, - 0x71, 0x0b, 0xa3, 0x30, 0xe9, 0xf1, 0x42, 0x90, 0xcf, 0x70, 0xe7, 0xf4, 0x9f, 0x19, 0x40, 0xf1, - 0x49, 0xfd, 0xb6, 0x29, 0xde, 0x3d, 0xa8, 0xb9, 0x9e, 0xe1, 0xc4, 0x36, 0x5b, 0x95, 0x51, 0x7d, - 0x8f, 0xff, 0x36, 0xf8, 0x0a, 0x9d, 0x58, 0xb6, 0x67, 0xbe, 0xbc, 0x12, 0x59, 0x72, 0x4d, 0x92, - 0x0f, 0x18, 0x15, 0x75, 0xa0, 0xf0, 0xd2, 0x1c, 0x78, 0xc4, 0x71, 0x1b, 0xb9, 0xf5, 0xec, 0x46, - 0x6d, 0xeb, 0xe1, 0x75, 0xcb, 0xb0, 0xf9, 0x21, 0xe3, 0xef, 0x5e, 0x8d, 0x88, 0x2e, 0xc7, 0xaa, - 0x99, 0x67, 0x3e, 0x94, 0x8d, 0xdf, 0x84, 0xe2, 0x2b, 0x2a, 0x82, 0xde, 0xb2, 0x0b, 0x3c, 0x59, - 0x64, 0x6d, 0x7e, 0xc9, 0x7e, 0xe9, 0x18, 0x67, 0x43, 0x62, 0x79, 0xf2, 0x1e, 0x28, 0xdb, 0xf8, - 0x1e, 0x40, 0x00, 0x43, 0x5d, 0xfe, 0xc1, 0xe1, 0xd1, 0xf3, 0x6e, 0x7d, 0x0e, 0x55, 0xa0, 0x78, - 0x70, 0xb8, 0xd3, 0xd9, 0xef, 0xd0, 0xf8, 0x80, 0x5b, 0xd2, 0xa4, 0xa1, 0xb5, 0x54, 0x31, 0xb5, - 0x10, 0x26, 0x5e, 0x85, 0xe5, 0xa4, 0x05, 0xa4, 0xb9, 0x68, 0x55, 0xec, 0xd2, 0x99, 0x8e, 0x8a, - 0x0a, 0x9d, 0x09, 0x4f, 0xb7, 0x01, 0x05, 0xbe, 0x7b, 0xfb, 0x22, 0x39, 0x97, 0x4d, 0x6a, 0x08, - 0xbe, 0x19, 0x49, 0x5f, 0xac, 0x92, 0xdf, 0x4e, 0x74, 0x2f, 0xb9, 0x44, 0xf7, 0x82, 0xee, 0x42, - 0xd5, 0x3f, 0x0d, 0x86, 0x2b, 0x72, 0x81, 0x92, 0x5e, 0x91, 0x1b, 0x9d, 0xd2, 0x42, 0x46, 0x2f, - 0x84, 0x8d, 0x8e, 0xee, 0x41, 0x9e, 0x4c, 0x88, 0xe5, 0xb9, 0x8d, 0x32, 0x8b, 0x18, 0x55, 0x99, - 0xbb, 0x77, 0x28, 0x55, 0x17, 0x9d, 0xf8, 0xbb, 0xb0, 0xc8, 0xee, 0x48, 0x4f, 0x1c, 0xc3, 0x52, - 0x2f, 0x73, 0xdd, 0xee, 0xbe, 0x30, 0x37, 0xfd, 0x89, 0x6a, 0x90, 0xd9, 0xdd, 0x11, 0x46, 0xc8, - 0xec, 0xee, 0xe0, 0x1f, 0x6b, 0x80, 0xd4, 0x71, 0x33, 0xd9, 0x39, 0x22, 0x5c, 0xc2, 0x67, 0x03, - 0xf8, 0x65, 0xc8, 0x11, 0xc7, 0xb1, 0x1d, 0x66, 0xd1, 0x92, 0xce, 0x1b, 0xf8, 0x2d, 0xa1, 0x83, - 0x4e, 0x26, 0xf6, 0x85, 0x7f, 0x06, 0xb9, 0x34, 0xcd, 0x57, 0x75, 0x0f, 0x96, 0x42, 0x5c, 0x33, - 0x45, 0xae, 0x0f, 0x61, 0x81, 0x09, 0xdb, 0x3e, 0x27, 0xbd, 0x8b, 0x91, 0x6d, 0x5a, 0x31, 0x3c, - 0xba, 0x72, 0x81, 0x83, 0xa5, 0xf3, 0xe0, 0x13, 0xab, 0xf8, 0xc4, 0x6e, 0x77, 0x1f, 0x7f, 0x06, - 0xab, 0x11, 0x39, 0x52, 0xfd, 0x3f, 0x82, 0x72, 0xcf, 0x27, 0xba, 0x22, 0xd7, 0xb9, 0x1d, 0x56, - 0x2e, 0x3a, 0x54, 0x1d, 0x81, 0x0f, 0xe1, 0x46, 0x4c, 0xf4, 0x4c, 0x73, 0x7e, 0x1b, 0x56, 0x98, - 0xc0, 0x3d, 0x42, 0x46, 0xed, 0x81, 0x39, 0x49, 0xb5, 0xf4, 0x48, 0x4c, 0x4a, 0x61, 0xfc, 0x7a, - 0xf7, 0x05, 0xfe, 0x03, 0x81, 0xd8, 0x35, 0x87, 0xa4, 0x6b, 0xef, 0xa7, 0xeb, 0x46, 0xa3, 0xd9, - 0x05, 0xb9, 0x72, 0x45, 0x5a, 0xc3, 0x7e, 0xe3, 0x7f, 0xd6, 0x84, 0xa9, 0xd4, 0xe1, 0x5f, 0xf3, - 0x4e, 0x5e, 0x03, 0x38, 0xa3, 0x47, 0x86, 0xf4, 0x69, 0x07, 0xaf, 0xa8, 0x28, 0x14, 0x5f, 0x4f, - 0xea, 0xbf, 0x2b, 0x42, 0xcf, 0x65, 0xb1, 0xcf, 0xd9, 0x3f, 0xbe, 0x97, 0xbb, 0x0d, 0x65, 0x46, - 0x38, 0xf6, 0x0c, 0x6f, 0xec, 0xc6, 0x16, 0xe3, 0x2f, 0xc4, 0xb6, 0x97, 0x83, 0x66, 0x9a, 0xd7, - 0xb7, 0x20, 0xcf, 0x2e, 0x13, 0x32, 0x95, 0xbe, 0x99, 0xb0, 0x1f, 0xb9, 0x1e, 0xba, 0x60, 0xc4, - 0x3f, 0xd5, 0x20, 0xff, 0x8c, 0x95, 0x60, 0x15, 0xd5, 0xe6, 0xe5, 0x5a, 0x58, 0xc6, 0x90, 0x17, - 0x86, 0x4a, 0x3a, 0xfb, 0xcd, 0x52, 0x4f, 0x42, 0x9c, 0xe7, 0xfa, 0x3e, 0x4f, 0x71, 0x4b, 0xba, - 0xdf, 0xa6, 0x36, 0xeb, 0x0d, 0x4c, 0x62, 0x79, 0xac, 0x77, 0x9e, 0xf5, 0x2a, 0x14, 0x9a, 0x3d, - 0x9b, 0xee, 0x3e, 0x31, 0x1c, 0x4b, 0x14, 0x4d, 0x8b, 0x7a, 0x40, 0xc0, 0xfb, 0x50, 0xe7, 0x7a, - 0xb4, 0xfb, 0x7d, 0x25, 0xc1, 0xf4, 0xd1, 0xb4, 0x08, 0x5a, 0x48, 0x5a, 0x26, 0x2a, 0xed, 0x17, - 0x1a, 0x2c, 0x2a, 0xe2, 0x66, 0xb2, 0xea, 0xbb, 0x90, 0xe7, 0x45, 0x6a, 0x91, 0xe9, 0x2c, 0x87, - 0x47, 0x71, 0x18, 0x5d, 0xf0, 0xa0, 0x4d, 0x28, 0xf0, 0x5f, 0xf2, 0x0e, 0x90, 0xcc, 0x2e, 0x99, - 0xf0, 0x3d, 0x58, 0x12, 0x24, 0x32, 0xb4, 0x93, 0x0e, 0x06, 0x5b, 0x0c, 0xfc, 0x67, 0xb0, 0x1c, - 0x66, 0x9b, 0x69, 0x4a, 0x8a, 0x92, 0x99, 0xd7, 0x51, 0xb2, 0x2d, 0x95, 0x7c, 0x3e, 0xea, 0x2b, - 0x79, 0x54, 0x74, 0xc7, 0xa8, 0xeb, 0x95, 0x09, 0xaf, 0x57, 0x30, 0x01, 0x29, 0xe2, 0x1b, 0x9d, - 0xc0, 0x07, 0x72, 0x3b, 0xec, 0x9b, 0xae, 0xef, 0xc3, 0x31, 0x54, 0x06, 0xa6, 0x45, 0x0c, 0x47, - 0x54, 0xce, 0x35, 0x5e, 0x39, 0x57, 0x69, 0xf8, 0x0b, 0x40, 0xea, 0xc0, 0x6f, 0x54, 0xe9, 0xfb, - 0xd2, 0x64, 0x47, 0x8e, 0x3d, 0xb4, 0x53, 0xcd, 0x8e, 0xff, 0x1c, 0x56, 0x22, 0x7c, 0xdf, 0xa8, - 0x9a, 0x4b, 0xb0, 0xb8, 0x43, 0x64, 0x42, 0x23, 0xdd, 0xde, 0x47, 0x80, 0x54, 0xe2, 0x4c, 0x91, - 0xad, 0x05, 0x8b, 0xcf, 0xec, 0x09, 0x75, 0x91, 0x94, 0x1a, 0xf8, 0x06, 0x5e, 0x87, 0xf0, 0x4d, - 0xe1, 0xb7, 0x29, 0xb8, 0x3a, 0x60, 0x26, 0xf0, 0x7f, 0xd7, 0xa0, 0xd2, 0x1e, 0x18, 0xce, 0x50, - 0x02, 0x7f, 0x0f, 0xf2, 0xfc, 0x76, 0x2d, 0x0a, 0x5a, 0xf7, 0xc3, 0x62, 0x54, 0x5e, 0xde, 0x68, - 0xf3, 0xbb, 0xb8, 0x18, 0x45, 0x15, 0x17, 0x6f, 0x5e, 0x3b, 0x91, 0x37, 0xb0, 0x1d, 0xf4, 0x1e, - 0xe4, 0x0c, 0x3a, 0x84, 0x85, 0xa2, 0x5a, 0xb4, 0xae, 0xc1, 0xa4, 0xb1, 0x3b, 0x00, 0xe7, 0xc2, - 0xdf, 0x81, 0xb2, 0x82, 0x80, 0x0a, 0x90, 0x7d, 0xd2, 0x11, 0x09, 0x7b, 0x7b, 0xbb, 0xbb, 0xfb, - 0x82, 0x17, 0x74, 0x6a, 0x00, 0x3b, 0x1d, 0xbf, 0x9d, 0xc1, 0x9f, 0x8a, 0x51, 0xc2, 0xed, 0xab, - 0xfa, 0x68, 0x69, 0xfa, 0x64, 0x5e, 0x4b, 0x9f, 0x4b, 0xa8, 0x8a, 0xe9, 0xcf, 0x1a, 0xc6, 0x98, - 0xbc, 0x94, 0x30, 0xa6, 0x28, 0xaf, 0x0b, 0x46, 0xfc, 0x2b, 0x0d, 0xea, 0x3b, 0xf6, 0x2b, 0xeb, - 0xcc, 0x31, 0xfa, 0xfe, 0x39, 0xf9, 0x30, 0xb2, 0x52, 0x9b, 0x91, 0xe2, 0x68, 0x84, 0x3f, 0x20, - 0x44, 0x56, 0xac, 0x11, 0x94, 0x0d, 0x79, 0x2c, 0x94, 0x4d, 0xfc, 0x01, 0x2c, 0x44, 0x06, 0x51, - 0xdb, 0xbf, 0x68, 0xef, 0xef, 0xee, 0x50, 0x5b, 0xb3, 0xc2, 0x5a, 0xe7, 0xa0, 0xfd, 0x78, 0xbf, - 0x23, 0x1e, 0x90, 0xda, 0x07, 0xdb, 0x9d, 0xfd, 0x7a, 0x06, 0xf7, 0x60, 0x51, 0x81, 0x9f, 0xf5, - 0x65, 0x20, 0x45, 0xbb, 0x05, 0xa8, 0x8a, 0x68, 0x2f, 0x0e, 0xe5, 0xbf, 0x65, 0xa0, 0x26, 0x29, - 0x5f, 0x0f, 0x26, 0x5a, 0x85, 0x7c, 0xff, 0xf4, 0xd8, 0xfc, 0x42, 0xbe, 0x1c, 0x89, 0x16, 0xa5, - 0x0f, 0x38, 0x0e, 0x7f, 0xbe, 0x15, 0x2d, 0x1a, 0xc6, 0x1d, 0xe3, 0xa5, 0xb7, 0x6b, 0xf5, 0xc9, - 0x25, 0x4b, 0x0a, 0xe6, 0xf5, 0x80, 0xc0, 0x2a, 0x4c, 0xe2, 0x99, 0x97, 0xdd, 0xac, 0x94, 0x67, - 0x5f, 0xf4, 0x00, 0xea, 0xf4, 0x77, 0x7b, 0x34, 0x1a, 0x98, 0xa4, 0xcf, 0x05, 0x14, 0x18, 0x4f, - 0x8c, 0x4e, 0xd1, 0xd9, 0x5d, 0xc4, 0x6d, 0x14, 0x59, 0x58, 0x12, 0x2d, 0xb4, 0x0e, 0x65, 0xae, - 0xdf, 0xae, 0xf5, 0xdc, 0x25, 0xec, 0xed, 0x33, 0xab, 0xab, 0xa4, 0x70, 0x9a, 0x01, 0xd1, 0x34, - 0x63, 0x09, 0x16, 0xdb, 0x63, 0xef, 0xbc, 0x63, 0xd1, 0x58, 0x21, 0xad, 0xbc, 0x0c, 0x88, 0x12, - 0x77, 0x4c, 0x57, 0xa5, 0x0a, 0xd6, 0xf0, 0x82, 0x74, 0x60, 0x89, 0x12, 0x89, 0xe5, 0x99, 0x3d, - 0x25, 0xae, 0xca, 0xcc, 0x4b, 0x8b, 0x64, 0x5e, 0x86, 0xeb, 0xbe, 0xb2, 0x9d, 0xbe, 0xb0, 0xb9, - 0xdf, 0xc6, 0xff, 0xa8, 0x71, 0xc8, 0xe7, 0x6e, 0x28, 0x7d, 0xfa, 0x2d, 0xc5, 0xa0, 0xf7, 0xa1, - 0x60, 0x8f, 0xd8, 0x0b, 0xbf, 0x28, 0xc3, 0xac, 0x6e, 0xf2, 0x6f, 0x02, 0x36, 0x85, 0xe0, 0x43, - 0xde, 0xab, 0x4b, 0x36, 0x74, 0x1f, 0x6a, 0xe7, 0x86, 0x7b, 0x4e, 0xfa, 0x47, 0x52, 0x26, 0xbf, - 0xf9, 0x45, 0xa8, 0x78, 0x23, 0xd0, 0xef, 0x09, 0xf1, 0xa6, 0xe8, 0x87, 0x1f, 0xc2, 0x8a, 0xe4, - 0x14, 0xaf, 0x13, 0x53, 0x98, 0x5f, 0xc1, 0x6d, 0xc9, 0xbc, 0x7d, 0x6e, 0x58, 0x67, 0x44, 0x02, - 0xfe, 0xae, 0x16, 0x88, 0xcf, 0x27, 0x9b, 0x38, 0x9f, 0xc7, 0xd0, 0xf0, 0xe7, 0xc3, 0x6e, 0xd6, - 0xf6, 0x40, 0x55, 0x74, 0xec, 0x8a, 0xf3, 0x54, 0xd2, 0xd9, 0x6f, 0x4a, 0x73, 0xec, 0x81, 0x9f, - 0x4a, 0xd3, 0xdf, 0x78, 0x1b, 0x6e, 0x4a, 0x19, 0xe2, 0xce, 0x1b, 0x16, 0x12, 0x53, 0x3c, 0x49, - 0x88, 0x30, 0x2c, 0x1d, 0x3a, 0x7d, 0xe1, 0x55, 0xce, 0xf0, 0x12, 0x30, 0x99, 0x9a, 0x22, 0x73, - 0x85, 0x6f, 0x4a, 0xaa, 0x98, 0x92, 0x2d, 0x49, 0x32, 0x15, 0xa0, 0x92, 0xc5, 0x82, 0x51, 0x72, - 0x6c, 0xc1, 0x62, 0xa2, 0x7f, 0x00, 0x6b, 0xbe, 0x12, 0xd4, 0x6e, 0x47, 0xc4, 0x19, 0x9a, 0xae, - 0xab, 0xd4, 0xbd, 0x93, 0x26, 0x7e, 0x1f, 0xe6, 0x47, 0x44, 0x04, 0xa1, 0xf2, 0x16, 0x92, 0x9b, - 0x52, 0x19, 0xcc, 0xfa, 0x71, 0x1f, 0xee, 0x48, 0xe9, 0xdc, 0xa2, 0x89, 0xe2, 0xa3, 0x4a, 0xc9, - 0x6a, 0x60, 0x26, 0xa5, 0x1a, 0x98, 0x8d, 0xbc, 0xc5, 0x7c, 0xc4, 0x0d, 0x29, 0xcf, 0xfc, 0x4c, - 0xc9, 0xc5, 0x1e, 0xb7, 0xa9, 0xef, 0x2a, 0x66, 0x12, 0xf6, 0xd7, 0xc2, 0x0b, 0x7c, 0x55, 0x1e, - 0x9e, 0xb0, 0x19, 0xca, 0x87, 0x0e, 0xd9, 0xa4, 0x59, 0x33, 0x5d, 0x00, 0x5d, 0xad, 0x85, 0xce, - 0xeb, 0x21, 0x1a, 0x3e, 0x85, 0xe5, 0xb0, 0x5f, 0x9b, 0x49, 0x97, 0x65, 0xc8, 0x79, 0xf6, 0x05, - 0x91, 0xb1, 0x86, 0x37, 0xa4, 0xed, 0x7c, 0x9f, 0x37, 0x93, 0xed, 0x8c, 0x40, 0x18, 0x3b, 0x1d, - 0xb3, 0xea, 0x4b, 0x37, 0x96, 0xbc, 0x03, 0xf1, 0x06, 0x3e, 0x80, 0xd5, 0xa8, 0x67, 0x9b, 0x49, - 0xe5, 0x17, 0xfc, 0x2c, 0x25, 0x39, 0xbf, 0x99, 0xe4, 0x7e, 0x1c, 0xf8, 0x25, 0xc5, 0xb7, 0xcd, - 0x24, 0x52, 0x87, 0x66, 0x92, 0xab, 0xfb, 0x2a, 0x8e, 0x8e, 0xef, 0xf9, 0x66, 0x12, 0xe6, 0x06, - 0xc2, 0x66, 0x5f, 0xfe, 0xc0, 0x5d, 0x65, 0xa7, 0xba, 0x2b, 0x71, 0x48, 0x02, 0x87, 0xfa, 0x35, - 0x6c, 0x3a, 0x81, 0x11, 0xf8, 0xf2, 0x59, 0x31, 0x68, 0x38, 0xf3, 0x31, 0x58, 0x43, 0x6e, 0x6c, - 0x35, 0x02, 0xcc, 0xb4, 0x18, 0x9f, 0x04, 0x6e, 0x3c, 0x16, 0x24, 0x66, 0x12, 0xfc, 0x29, 0xac, - 0xa7, 0xc7, 0x87, 0x59, 0x24, 0x3f, 0x68, 0x41, 0xc9, 0xbf, 0x0c, 0x29, 0xdf, 0x9b, 0x95, 0xa1, - 0x70, 0x70, 0x78, 0x7c, 0xd4, 0xde, 0xee, 0xf0, 0x0f, 0xce, 0xb6, 0x0f, 0x75, 0xfd, 0xf9, 0x51, - 0xb7, 0x9e, 0xd9, 0xfa, 0x75, 0x16, 0x32, 0x7b, 0x2f, 0xd0, 0x67, 0x90, 0xe3, 0x5f, 0x5f, 0x4c, - 0xf9, 0xe4, 0xa6, 0x39, 0xed, 0x03, 0x13, 0x7c, 0xe3, 0xc7, 0xff, 0xf5, 0xeb, 0x9f, 0x67, 0x16, - 0x71, 0xa5, 0x35, 0xf9, 0x76, 0xeb, 0x62, 0xd2, 0x62, 0x61, 0xea, 0x91, 0xf6, 0x00, 0x7d, 0x0c, - 0xd9, 0xa3, 0xb1, 0x87, 0x52, 0x3f, 0xc5, 0x69, 0xa6, 0x7f, 0x73, 0x82, 0x57, 0x98, 0xd0, 0x05, - 0x0c, 0x42, 0xe8, 0x68, 0xec, 0x51, 0x91, 0x3f, 0x84, 0xb2, 0xfa, 0xc5, 0xc8, 0xb5, 0xdf, 0xe7, - 0x34, 0xaf, 0xff, 0x1a, 0x05, 0xdf, 0x66, 0x50, 0x37, 0x30, 0x12, 0x50, 0xfc, 0x9b, 0x16, 0x75, - 0x16, 0xdd, 0x4b, 0x0b, 0xa5, 0x7e, 0xbd, 0xd3, 0x4c, 0xff, 0x40, 0x25, 0x36, 0x0b, 0xef, 0xd2, - 0xa2, 0x22, 0xff, 0x44, 0x7c, 0x9b, 0xd2, 0xf3, 0xd0, 0x9d, 0x84, 0x6f, 0x13, 0xd4, 0x57, 0xf8, - 0xe6, 0x7a, 0x3a, 0x83, 0x00, 0xb9, 0xc5, 0x40, 0x56, 0xf1, 0xa2, 0x00, 0xe9, 0xf9, 0x2c, 0x8f, - 0xb4, 0x07, 0x5b, 0x3d, 0xc8, 0xb1, 0x17, 0x2e, 0xf4, 0xb9, 0xfc, 0xd1, 0x4c, 0x78, 0xea, 0x4b, - 0x59, 0xe8, 0xd0, 0xdb, 0x18, 0x5e, 0x66, 0x40, 0x35, 0x5c, 0xa2, 0x40, 0xec, 0x7d, 0xeb, 0x91, - 0xf6, 0x60, 0x43, 0x7b, 0x5f, 0xdb, 0xfa, 0x55, 0x0e, 0x72, 0xac, 0xb4, 0x8b, 0x2e, 0x00, 0x82, - 0xd7, 0x9e, 0xe8, 0xec, 0x62, 0xef, 0x47, 0xd1, 0xd9, 0xc5, 0x1f, 0x8a, 0x70, 0x93, 0x81, 0x2e, - 0xe3, 0x05, 0x0a, 0xca, 0x2a, 0xc6, 0x2d, 0x56, 0x04, 0xa7, 0x76, 0xfc, 0x1b, 0x4d, 0x54, 0xb6, - 0xf9, 0x59, 0x42, 0x49, 0xd2, 0x42, 0x4f, 0x3e, 0xd1, 0xed, 0x90, 0xf0, 0xdc, 0x83, 0xbf, 0xcb, - 0x00, 0x5b, 0xb8, 0x1e, 0x00, 0x3a, 0x8c, 0xe3, 0x91, 0xf6, 0xe0, 0xf3, 0x06, 0x5e, 0x12, 0x56, - 0x8e, 0xf4, 0xa0, 0x1f, 0x41, 0x2d, 0xfc, 0xa4, 0x81, 0xee, 0x26, 0x60, 0x45, 0x5f, 0x46, 0x9a, - 0x6f, 0x4d, 0x67, 0x12, 0x3a, 0xad, 0x31, 0x9d, 0x04, 0x38, 0x47, 0xbe, 0x20, 0x64, 0x64, 0x50, - 0x26, 0xb1, 0x06, 0xe8, 0x1f, 0x34, 0xf1, 0xe2, 0x14, 0xbc, 0x51, 0xa0, 0x24, 0xe9, 0xb1, 0x17, - 0x90, 0xe6, 0xbd, 0x6b, 0xb8, 0x84, 0x12, 0x7f, 0xc8, 0x94, 0xf8, 0x00, 0x2f, 0x07, 0x4a, 0x78, - 0xe6, 0x90, 0x78, 0xb6, 0xd0, 0xe2, 0xf3, 0x5b, 0xf8, 0x46, 0xc8, 0x38, 0xa1, 0xde, 0x60, 0xb1, - 0xf8, 0x3b, 0x43, 0xe2, 0x62, 0x85, 0xde, 0x2d, 0x12, 0x17, 0x2b, 0xfc, 0x48, 0x91, 0xb4, 0x58, - 0xfc, 0x55, 0x21, 0x69, 0xb1, 0xfc, 0x9e, 0xad, 0xff, 0x9b, 0x87, 0xc2, 0x36, 0xff, 0x26, 0x1c, - 0xd9, 0x50, 0xf2, 0xcb, 0xf4, 0x68, 0x2d, 0xa9, 0xce, 0x18, 0x5c, 0x6b, 0x9a, 0x77, 0x52, 0xfb, - 0x85, 0x42, 0x6f, 0x32, 0x85, 0xde, 0xc0, 0xab, 0x14, 0x59, 0x7c, 0x76, 0xde, 0xe2, 0xc5, 0xac, - 0x96, 0xd1, 0xef, 0x53, 0x43, 0xfc, 0x29, 0x54, 0xd4, 0x3a, 0x3a, 0x7a, 0x33, 0xb1, 0xb6, 0xa9, - 0x96, 0xe2, 0x9b, 0x78, 0x1a, 0x8b, 0x40, 0x7e, 0x8b, 0x21, 0xaf, 0xe1, 0x9b, 0x09, 0xc8, 0x0e, - 0x63, 0x0d, 0x81, 0xf3, 0x1a, 0x78, 0x32, 0x78, 0xa8, 0xc4, 0x9e, 0x0c, 0x1e, 0x2e, 0xa1, 0x4f, - 0x05, 0x1f, 0x33, 0x56, 0x0a, 0xee, 0x02, 0x04, 0x95, 0x6c, 0x94, 0x68, 0x4b, 0xe5, 0x5e, 0x17, - 0x75, 0x0e, 0xf1, 0x22, 0x38, 0xc6, 0x0c, 0x56, 0xec, 0xbb, 0x08, 0xec, 0xc0, 0x74, 0x3d, 0x7e, - 0x30, 0xab, 0xa1, 0xd2, 0x34, 0x4a, 0x9c, 0x4f, 0xb8, 0xbe, 0xdd, 0xbc, 0x3b, 0x95, 0x47, 0xa0, - 0xdf, 0x63, 0xe8, 0x77, 0x70, 0x33, 0x01, 0x7d, 0xc4, 0x79, 0xe9, 0x66, 0xfb, 0xff, 0x3c, 0x94, - 0x9f, 0x19, 0xa6, 0xe5, 0x11, 0xcb, 0xb0, 0x7a, 0x04, 0x9d, 0x42, 0x8e, 0x45, 0xea, 0xa8, 0x23, - 0x56, 0xcb, 0xb6, 0x51, 0x47, 0x1c, 0xaa, 0x69, 0xe2, 0x75, 0x06, 0xdc, 0xc4, 0x2b, 0x14, 0x78, - 0x18, 0x88, 0x6e, 0xb1, 0x52, 0x24, 0x9d, 0xf4, 0x4b, 0xc8, 0x8b, 0xd7, 0xbe, 0x88, 0xa0, 0x50, - 0xf1, 0xa7, 0x79, 0x2b, 0xb9, 0x33, 0x69, 0x2f, 0xab, 0x30, 0x2e, 0xe3, 0xa3, 0x38, 0x13, 0x80, - 0xa0, 0xc6, 0x1e, 0x5d, 0xd1, 0x58, 0x49, 0xbe, 0xb9, 0x9e, 0xce, 0x90, 0x64, 0x53, 0x15, 0xb3, - 0xef, 0xf3, 0x52, 0xdc, 0x3f, 0x86, 0xf9, 0xa7, 0x86, 0x7b, 0x8e, 0x22, 0xb1, 0x57, 0xf9, 0x56, - 0xac, 0xd9, 0x4c, 0xea, 0x12, 0x28, 0x77, 0x18, 0xca, 0x4d, 0xee, 0xca, 0x54, 0x94, 0x73, 0xc3, - 0xa5, 0x41, 0x0d, 0xf5, 0x21, 0xcf, 0x3f, 0x1d, 0x8b, 0xda, 0x2f, 0xf4, 0xf9, 0x59, 0xd4, 0x7e, - 0xe1, 0xaf, 0xcd, 0xae, 0x47, 0x19, 0x41, 0x51, 0x7e, 0xab, 0x85, 0x22, 0x0f, 0xf7, 0x91, 0xef, - 0xba, 0x9a, 0x6b, 0x69, 0xdd, 0x02, 0xeb, 0x2e, 0xc3, 0xba, 0x8d, 0x1b, 0xb1, 0xb5, 0x12, 0x9c, - 0x8f, 0xb4, 0x07, 0xef, 0x6b, 0xe8, 0x47, 0x00, 0xc1, 0xb3, 0x44, 0xec, 0x04, 0x46, 0x5f, 0x38, - 0x62, 0x27, 0x30, 0xf6, 0xa2, 0x81, 0x37, 0x19, 0xee, 0x06, 0xbe, 0x1b, 0xc5, 0xf5, 0x1c, 0xc3, - 0x72, 0x5f, 0x12, 0xe7, 0x3d, 0x5e, 0x65, 0x75, 0xcf, 0xcd, 0x11, 0x9d, 0xb2, 0x03, 0x25, 0xbf, - 0xea, 0x1c, 0xf5, 0xb6, 0xd1, 0x6a, 0x78, 0xd4, 0xdb, 0xc6, 0xca, 0xd5, 0x61, 0xb7, 0x13, 0xda, - 0x2d, 0x92, 0x95, 0x1e, 0xc0, 0x5f, 0xd4, 0x61, 0x9e, 0x66, 0xdd, 0x34, 0x39, 0x09, 0xea, 0x26, - 0xd1, 0xd9, 0xc7, 0xaa, 0xa8, 0xd1, 0xd9, 0xc7, 0x4b, 0x2e, 0xe1, 0xe4, 0x84, 0x5e, 0xb2, 0x5a, - 0xbc, 0x44, 0x41, 0x67, 0x6a, 0x43, 0x59, 0x29, 0xac, 0xa0, 0x04, 0x61, 0xe1, 0xf2, 0x6c, 0x34, - 0xdc, 0x25, 0x54, 0x65, 0xf0, 0x1b, 0x0c, 0x6f, 0x85, 0x87, 0x3b, 0x86, 0xd7, 0xe7, 0x1c, 0x14, - 0x50, 0xcc, 0x4e, 0x9c, 0xfb, 0x84, 0xd9, 0x85, 0xcf, 0xfe, 0x7a, 0x3a, 0x43, 0xea, 0xec, 0x82, - 0x83, 0xff, 0x0a, 0x2a, 0x6a, 0x79, 0x05, 0x25, 0x28, 0x1f, 0x29, 0x29, 0x47, 0xe3, 0x48, 0x52, - 0x75, 0x26, 0xec, 0xd9, 0x18, 0xa4, 0xa1, 0xb0, 0x51, 0xe0, 0x01, 0x14, 0x44, 0xbd, 0x25, 0xc9, - 0xa4, 0xe1, 0xf2, 0x73, 0x92, 0x49, 0x23, 0xc5, 0x9a, 0x70, 0xf6, 0xcc, 0x10, 0xe9, 0x95, 0x52, - 0xc6, 0x6a, 0x81, 0xf6, 0x84, 0x78, 0x69, 0x68, 0x41, 0x25, 0x33, 0x0d, 0x4d, 0xb9, 0xce, 0xa7, - 0xa1, 0x9d, 0x11, 0x4f, 0xf8, 0x03, 0x79, 0x4d, 0x46, 0x29, 0xc2, 0xd4, 0xf8, 0x88, 0xa7, 0xb1, - 0x24, 0x5d, 0x6e, 0x02, 0x40, 0x19, 0x1c, 0x2f, 0x01, 0x82, 0x6a, 0x50, 0x34, 0x63, 0x4d, 0xac, - 0x82, 0x47, 0x33, 0xd6, 0xe4, 0x82, 0x52, 0xd8, 0xf7, 0x05, 0xb8, 0xfc, 0x6e, 0x45, 0x91, 0x7f, - 0xa6, 0x01, 0x8a, 0x17, 0x8e, 0xd0, 0xc3, 0x64, 0xe9, 0x89, 0xb5, 0xf5, 0xe6, 0xbb, 0xaf, 0xc7, - 0x9c, 0x14, 0xce, 0x02, 0x95, 0x7a, 0x8c, 0x7b, 0xf4, 0x8a, 0x2a, 0xf5, 0x97, 0x1a, 0x54, 0x43, - 0x55, 0x27, 0x74, 0x3f, 0x65, 0x4d, 0x23, 0x25, 0xf7, 0xe6, 0xdb, 0xd7, 0xf2, 0x25, 0xa5, 0xf2, - 0xca, 0x0e, 0x90, 0x77, 0x9a, 0x9f, 0x68, 0x50, 0x0b, 0x57, 0xa9, 0x50, 0x8a, 0xec, 0x58, 0xc9, - 0xbe, 0xb9, 0x71, 0x3d, 0xe3, 0xf4, 0xe5, 0x09, 0xae, 0x33, 0x03, 0x28, 0x88, 0xba, 0x56, 0xd2, - 0xc6, 0x0f, 0x17, 0xfb, 0x93, 0x36, 0x7e, 0xa4, 0x28, 0x96, 0xb0, 0xf1, 0x1d, 0x7b, 0x40, 0x94, - 0x63, 0x26, 0x0a, 0x5f, 0x69, 0x68, 0xd3, 0x8f, 0x59, 0xa4, 0x6a, 0x96, 0x86, 0x16, 0x1c, 0x33, - 0x59, 0xf1, 0x42, 0x29, 0xc2, 0xae, 0x39, 0x66, 0xd1, 0x82, 0x59, 0xc2, 0x31, 0x63, 0x80, 0xca, - 0x31, 0x0b, 0x6a, 0x53, 0x49, 0xc7, 0x2c, 0xf6, 0x76, 0x91, 0x74, 0xcc, 0xe2, 0xe5, 0xad, 0x84, - 0x75, 0x64, 0xb8, 0xa1, 0x63, 0xb6, 0x94, 0x50, 0xc6, 0x42, 0xef, 0xa6, 0x18, 0x31, 0xf1, 0x49, - 0xa4, 0xf9, 0xde, 0x6b, 0x72, 0xa7, 0xee, 0x71, 0x6e, 0x7e, 0xb9, 0xc7, 0xff, 0x56, 0x83, 0xe5, - 0xa4, 0x12, 0x18, 0x4a, 0xc1, 0x49, 0x79, 0x4a, 0x69, 0x6e, 0xbe, 0x2e, 0xfb, 0x74, 0x6b, 0xf9, - 0xbb, 0xfe, 0x71, 0xfd, 0x5f, 0xbf, 0x5c, 0xd3, 0xfe, 0xe3, 0xcb, 0x35, 0xed, 0xbf, 0xbf, 0x5c, - 0xd3, 0xfe, 0xee, 0x7f, 0xd6, 0xe6, 0x4e, 0xf3, 0xec, 0x3f, 0x1a, 0x7f, 0xfb, 0x37, 0x01, 0x00, - 0x00, 0xff, 0xff, 0xee, 0x4f, 0x63, 0x90, 0xed, 0x3c, 0x00, 0x00, + // 4110 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0xdd, 0x73, 0x1b, 0xc9, + 0x71, 0xe7, 0x02, 0x24, 0x01, 0x34, 0x3e, 0x08, 0x0e, 0x29, 0x0a, 0xc2, 0x49, 0x14, 0x6f, 0x74, + 0xd2, 0xf1, 0xa4, 0x3b, 0xe2, 0x4c, 0xdb, 0xb9, 0x2a, 0x25, 0x71, 0x0c, 0x91, 0x38, 0x89, 0x47, + 0x8a, 0xe4, 0x2d, 0x21, 0xdd, 0x47, 0xb9, 0xc2, 0x5a, 0x02, 0x23, 0x72, 0x43, 0x60, 0x17, 0xde, + 0x5d, 0x40, 0xe4, 0xe5, 0xc3, 0x2e, 0x97, 0xe3, 0x4a, 0x5e, 0xed, 0xaa, 0x54, 0xf2, 0x90, 0xbc, + 0xa4, 0x52, 0x2e, 0x3f, 0xf8, 0x39, 0xff, 0x42, 0x9e, 0xf2, 0x51, 0xf9, 0x07, 0x52, 0x17, 0xbf, + 0x24, 0x7f, 0x85, 0x6b, 0xbe, 0x76, 0x67, 0xf6, 0x83, 0x92, 0x8d, 0xbb, 0x7b, 0x11, 0x31, 0x3d, + 0x3d, 0xfd, 0xeb, 0xe9, 0x99, 0xe9, 0xee, 0xe9, 0x59, 0x41, 0xc9, 0x1b, 0xf5, 0x36, 0x46, 0x9e, + 0x1b, 0xb8, 0xa8, 0x42, 0x82, 0x5e, 0xdf, 0x27, 0xde, 0x84, 0x78, 0xa3, 0x93, 0xe6, 0xf2, 0xa9, + 0x7b, 0xea, 0xb2, 0x8e, 0x16, 0xfd, 0xc5, 0x79, 0x9a, 0x0d, 0xca, 0xd3, 0xb2, 0x46, 0x76, 0x6b, + 0x38, 0xe9, 0xf5, 0x46, 0x27, 0xad, 0xf3, 0x89, 0xe8, 0x69, 0x86, 0x3d, 0xd6, 0x38, 0x38, 0x1b, + 0x9d, 0xb0, 0x3f, 0xa2, 0xef, 0xe6, 0xa9, 0xeb, 0x9e, 0x0e, 0x08, 0xef, 0x75, 0x1c, 0x37, 0xb0, + 0x02, 0xdb, 0x75, 0x7c, 0xde, 0x8b, 0xff, 0xda, 0x80, 0x9a, 0x49, 0xfc, 0x91, 0xeb, 0xf8, 0xe4, + 0x09, 0xb1, 0xfa, 0xc4, 0x43, 0xb7, 0x00, 0x7a, 0x83, 0xb1, 0x1f, 0x10, 0xef, 0xd8, 0xee, 0x37, + 0x8c, 0x35, 0x63, 0x7d, 0xd6, 0x2c, 0x09, 0xca, 0x4e, 0x1f, 0xbd, 0x01, 0xa5, 0x21, 0x19, 0x9e, + 0xf0, 0xde, 0x1c, 0xeb, 0x2d, 0x72, 0xc2, 0x4e, 0x1f, 0x35, 0xa1, 0xe8, 0x91, 0x89, 0xed, 0xdb, + 0xae, 0xd3, 0xc8, 0xaf, 0x19, 0xeb, 0x79, 0x33, 0x6c, 0xd3, 0x81, 0x9e, 0xf5, 0x22, 0x38, 0x0e, + 0x88, 0x37, 0x6c, 0xcc, 0xf2, 0x81, 0x94, 0xd0, 0x25, 0xde, 0x10, 0xff, 0x74, 0x0e, 0x2a, 0xa6, + 0xe5, 0x9c, 0x12, 0x93, 0xfc, 0x70, 0x4c, 0xfc, 0x00, 0xd5, 0x21, 0x7f, 0x4e, 0x2e, 0x19, 0x7c, + 0xc5, 0xa4, 0x3f, 0xf9, 0x78, 0xe7, 0x94, 0x1c, 0x13, 0x87, 0x03, 0x57, 0xe8, 0x78, 0xe7, 0x94, + 0x74, 0x9c, 0x3e, 0x5a, 0x86, 0xb9, 0x81, 0x3d, 0xb4, 0x03, 0x81, 0xca, 0x1b, 0x9a, 0x3a, 0xb3, + 0x31, 0x75, 0xb6, 0x00, 0x7c, 0xd7, 0x0b, 0x8e, 0x5d, 0xaf, 0x4f, 0xbc, 0xc6, 0xdc, 0x9a, 0xb1, + 0x5e, 0xdb, 0x7c, 0x6b, 0x43, 0x5d, 0x86, 0x0d, 0x55, 0xa1, 0x8d, 0x23, 0xd7, 0x0b, 0x0e, 0x28, + 0xaf, 0x59, 0xf2, 0xe5, 0x4f, 0xf4, 0x21, 0x94, 0x99, 0x90, 0xc0, 0xf2, 0x4e, 0x49, 0xd0, 0x98, + 0x67, 0x52, 0xee, 0xbe, 0x42, 0x4a, 0x97, 0x31, 0x9b, 0x0c, 0x9e, 0xff, 0x46, 0x18, 0x2a, 0x3e, + 0xf1, 0x6c, 0x6b, 0x60, 0x7f, 0x61, 0x9d, 0x0c, 0x48, 0xa3, 0xb0, 0x66, 0xac, 0x17, 0x4d, 0x8d, + 0x46, 0xe7, 0x7f, 0x4e, 0x2e, 0xfd, 0x63, 0xd7, 0x19, 0x5c, 0x36, 0x8a, 0x8c, 0xa1, 0x48, 0x09, + 0x07, 0xce, 0xe0, 0x92, 0x2d, 0x9a, 0x3b, 0x76, 0x02, 0xde, 0x5b, 0x62, 0xbd, 0x25, 0x46, 0x61, + 0xdd, 0xeb, 0x50, 0x1f, 0xda, 0xce, 0xf1, 0xd0, 0xed, 0x1f, 0x87, 0x06, 0x01, 0x66, 0x90, 0xda, + 0xd0, 0x76, 0x9e, 0xba, 0x7d, 0x53, 0x9a, 0x85, 0x72, 0x5a, 0x17, 0x3a, 0x67, 0x59, 0x70, 0x5a, + 0x17, 0x2a, 0xe7, 0x06, 0x2c, 0x51, 0x99, 0x3d, 0x8f, 0x58, 0x01, 0x89, 0x98, 0x2b, 0x8c, 0x79, + 0x71, 0x68, 0x3b, 0x5b, 0xac, 0x47, 0xe3, 0xb7, 0x2e, 0x12, 0xfc, 0x55, 0xc1, 0x6f, 0x5d, 0xe8, + 0xfc, 0x78, 0x03, 0x4a, 0xa1, 0xcd, 0x51, 0x11, 0x66, 0xf7, 0x0f, 0xf6, 0x3b, 0xf5, 0x19, 0x04, + 0x30, 0xdf, 0x3e, 0xda, 0xea, 0xec, 0x6f, 0xd7, 0x0d, 0x54, 0x86, 0xc2, 0x76, 0x87, 0x37, 0x72, + 0xf8, 0x11, 0x40, 0x64, 0x5d, 0x54, 0x80, 0xfc, 0x6e, 0xe7, 0xb3, 0xfa, 0x0c, 0xe5, 0x79, 0xde, + 0x31, 0x8f, 0x76, 0x0e, 0xf6, 0xeb, 0x06, 0x1d, 0xbc, 0x65, 0x76, 0xda, 0xdd, 0x4e, 0x3d, 0x47, + 0x39, 0x9e, 0x1e, 0x6c, 0xd7, 0xf3, 0xa8, 0x04, 0x73, 0xcf, 0xdb, 0x7b, 0xcf, 0x3a, 0xf5, 0x59, + 0xfc, 0x0b, 0x03, 0xaa, 0x62, 0xbd, 0xf8, 0x99, 0x40, 0xdf, 0x81, 0xf9, 0x33, 0x76, 0x2e, 0xd8, + 0x56, 0x2c, 0x6f, 0xde, 0x8c, 0x2d, 0xae, 0x76, 0x76, 0x4c, 0xc1, 0x8b, 0x30, 0xe4, 0xcf, 0x27, + 0x7e, 0x23, 0xb7, 0x96, 0x5f, 0x2f, 0x6f, 0xd6, 0x37, 0xf8, 0x79, 0xdd, 0xd8, 0x25, 0x97, 0xcf, + 0xad, 0xc1, 0x98, 0x98, 0xb4, 0x13, 0x21, 0x98, 0x1d, 0xba, 0x1e, 0x61, 0x3b, 0xb6, 0x68, 0xb2, + 0xdf, 0x74, 0x1b, 0xb3, 0x45, 0x13, 0xbb, 0x95, 0x37, 0xf0, 0xaf, 0x0c, 0x80, 0xc3, 0x71, 0x90, + 0x7d, 0x34, 0x96, 0x61, 0x6e, 0x42, 0x05, 0x8b, 0x63, 0xc1, 0x1b, 0xec, 0x4c, 0x10, 0xcb, 0x27, + 0xe1, 0x99, 0xa0, 0x0d, 0x74, 0x1d, 0x0a, 0x23, 0x8f, 0x4c, 0x8e, 0xcf, 0x27, 0x0c, 0xa4, 0x68, + 0xce, 0xd3, 0xe6, 0xee, 0x04, 0xbd, 0x09, 0x15, 0xfb, 0xd4, 0x71, 0x3d, 0x72, 0xcc, 0x65, 0xcd, + 0xb1, 0xde, 0x32, 0xa7, 0x31, 0xbd, 0x15, 0x16, 0x2e, 0x78, 0x5e, 0x65, 0xd9, 0xa3, 0x24, 0xec, + 0x40, 0x99, 0xa9, 0x3a, 0x95, 0xf9, 0xde, 0x89, 0x74, 0xcc, 0xb1, 0x61, 0x49, 0x13, 0x0a, 0xad, + 0xf1, 0x0f, 0x00, 0x6d, 0x93, 0x01, 0x09, 0xc8, 0x34, 0xde, 0x43, 0xb1, 0x49, 0x5e, 0xb5, 0x09, + 0xfe, 0xb9, 0x01, 0x4b, 0x9a, 0xf8, 0xa9, 0xa6, 0xd5, 0x80, 0x42, 0x9f, 0x09, 0xe3, 0x1a, 0xe4, + 0x4d, 0xd9, 0x44, 0x0f, 0xa0, 0x28, 0x14, 0xf0, 0x1b, 0xf9, 0x8c, 0x4d, 0x53, 0xe0, 0x3a, 0xf9, + 0xf8, 0x57, 0x39, 0x28, 0x89, 0x89, 0x1e, 0x8c, 0x50, 0x1b, 0xaa, 0x1e, 0x6f, 0x1c, 0xb3, 0xf9, + 0x08, 0x8d, 0x9a, 0xd9, 0x4e, 0xe8, 0xc9, 0x8c, 0x59, 0x11, 0x43, 0x18, 0x19, 0xfd, 0x21, 0x94, + 0xa5, 0x88, 0xd1, 0x38, 0x10, 0x26, 0x6f, 0xe8, 0x02, 0xa2, 0xfd, 0xf7, 0x64, 0xc6, 0x04, 0xc1, + 0x7e, 0x38, 0x0e, 0x50, 0x17, 0x96, 0xe5, 0x60, 0x3e, 0x1b, 0xa1, 0x46, 0x9e, 0x49, 0x59, 0xd3, + 0xa5, 0x24, 0x97, 0xea, 0xc9, 0x8c, 0x89, 0xc4, 0x78, 0xa5, 0x53, 0x55, 0x29, 0xb8, 0xe0, 0xce, + 0x3b, 0xa1, 0x52, 0xf7, 0xc2, 0x49, 0xaa, 0xd4, 0xbd, 0x70, 0x1e, 0x95, 0xa0, 0x20, 0x5a, 0xf8, + 0x5f, 0x73, 0x00, 0x72, 0x35, 0x0e, 0x46, 0x68, 0x1b, 0x6a, 0x9e, 0x68, 0x69, 0xd6, 0x7a, 0x23, + 0xd5, 0x5a, 0x62, 0x11, 0x67, 0xcc, 0xaa, 0x1c, 0xc4, 0x95, 0xfb, 0x1e, 0x54, 0x42, 0x29, 0x91, + 0xc1, 0x6e, 0xa4, 0x18, 0x2c, 0x94, 0x50, 0x96, 0x03, 0xa8, 0xc9, 0x3e, 0x81, 0x6b, 0xe1, 0xf8, + 0x14, 0x9b, 0xbd, 0x79, 0x85, 0xcd, 0x42, 0x81, 0x4b, 0x52, 0x82, 0x6a, 0x35, 0x55, 0xb1, 0xc8, + 0x6c, 0x37, 0x52, 0xcc, 0x96, 0x54, 0x8c, 0x1a, 0x0e, 0x68, 0xbc, 0xe4, 0x4d, 0xfc, 0x7f, 0x79, + 0x28, 0x6c, 0xb9, 0xc3, 0x91, 0xe5, 0xd1, 0xd5, 0x98, 0xf7, 0x88, 0x3f, 0x1e, 0x04, 0xcc, 0x5c, + 0xb5, 0xcd, 0x3b, 0xba, 0x44, 0xc1, 0x26, 0xff, 0x9a, 0x8c, 0xd5, 0x14, 0x43, 0xe8, 0x60, 0x11, + 0x1e, 0x73, 0xaf, 0x31, 0x58, 0x04, 0x47, 0x31, 0x44, 0x1e, 0xe4, 0x7c, 0x74, 0x90, 0x9b, 0x50, + 0x98, 0x10, 0x2f, 0x0a, 0xe9, 0x4f, 0x66, 0x4c, 0x49, 0x40, 0xef, 0xc0, 0x42, 0x3c, 0xbc, 0xcc, + 0x09, 0x9e, 0x5a, 0x4f, 0x8f, 0x46, 0x77, 0xa0, 0xa2, 0xc5, 0xb8, 0x79, 0xc1, 0x57, 0x1e, 0x2a, + 0x21, 0x6e, 0x45, 0xfa, 0x55, 0x1a, 0x8f, 0x2b, 0x4f, 0x66, 0xa4, 0x67, 0x5d, 0x91, 0x9e, 0xb5, + 0x28, 0x46, 0x09, 0xdf, 0xaa, 0x39, 0x99, 0xef, 0xeb, 0x4e, 0x06, 0x7f, 0x1f, 0xaa, 0x9a, 0x81, + 0x68, 0xdc, 0xe9, 0x7c, 0xfc, 0xac, 0xbd, 0xc7, 0x83, 0xd4, 0x63, 0x16, 0x97, 0xcc, 0xba, 0x41, + 0x63, 0xdd, 0x5e, 0xe7, 0xe8, 0xa8, 0x9e, 0x43, 0x55, 0x28, 0xed, 0x1f, 0x74, 0x8f, 0x39, 0x57, + 0x1e, 0x3f, 0x0e, 0x25, 0x88, 0x20, 0xa7, 0xc4, 0xb6, 0x19, 0x25, 0xb6, 0x19, 0x32, 0xb6, 0xe5, + 0xa2, 0xd8, 0xc6, 0xc2, 0xdc, 0x5e, 0xa7, 0x7d, 0xd4, 0xa9, 0xcf, 0x3e, 0xaa, 0x41, 0x85, 0xdb, + 0xf7, 0x78, 0xec, 0xd0, 0x50, 0xfb, 0xcf, 0x06, 0x40, 0x74, 0x9a, 0x50, 0x0b, 0x0a, 0x3d, 0x8e, + 0xd3, 0x30, 0x98, 0x33, 0xba, 0x96, 0xba, 0x64, 0xa6, 0xe4, 0x42, 0xdf, 0x82, 0x82, 0x3f, 0xee, + 0xf5, 0x88, 0x2f, 0x43, 0xde, 0xf5, 0xb8, 0x3f, 0x14, 0xde, 0xca, 0x94, 0x7c, 0x74, 0xc8, 0x0b, + 0xcb, 0x1e, 0x8c, 0x59, 0x00, 0xbc, 0x7a, 0x88, 0xe0, 0xc3, 0xff, 0x60, 0x40, 0x59, 0xd9, 0xbc, + 0xbf, 0xa7, 0x13, 0xbe, 0x09, 0x25, 0xa6, 0x03, 0xe9, 0x0b, 0x37, 0x5c, 0x34, 0x23, 0x02, 0xfa, + 0x03, 0x28, 0xc9, 0x13, 0x20, 0x3d, 0x71, 0x23, 0x5d, 0xec, 0xc1, 0xc8, 0x8c, 0x58, 0xf1, 0x2e, + 0x2c, 0x32, 0xab, 0xf4, 0x68, 0x72, 0x2d, 0xed, 0xa8, 0xa6, 0x9f, 0x46, 0x2c, 0xfd, 0x6c, 0x42, + 0x71, 0x74, 0x76, 0xe9, 0xdb, 0x3d, 0x6b, 0x20, 0xb4, 0x08, 0xdb, 0xf8, 0x23, 0x40, 0xaa, 0xb0, + 0x69, 0xa6, 0x8b, 0xab, 0x50, 0x7e, 0x62, 0xf9, 0x67, 0x42, 0x25, 0xfc, 0x00, 0xaa, 0xb4, 0xb9, + 0xfb, 0xfc, 0x35, 0x74, 0x64, 0x97, 0x03, 0xc9, 0x3d, 0x95, 0xcd, 0x11, 0xcc, 0x9e, 0x59, 0xfe, + 0x19, 0x9b, 0x68, 0xd5, 0x64, 0xbf, 0xd1, 0x3b, 0x50, 0xef, 0xf1, 0x49, 0x1e, 0xc7, 0xae, 0x0c, + 0x0b, 0x82, 0x1e, 0x66, 0x82, 0x9f, 0x42, 0x85, 0xcf, 0xe1, 0xab, 0x56, 0x02, 0x2f, 0xc2, 0xc2, + 0x91, 0x63, 0x8d, 0xfc, 0x33, 0x57, 0x46, 0x37, 0x3a, 0xe9, 0x7a, 0x44, 0x9b, 0x0a, 0xf1, 0x6d, + 0x58, 0xf0, 0xc8, 0xd0, 0xb2, 0x1d, 0xdb, 0x39, 0x3d, 0x3e, 0xb9, 0x0c, 0x88, 0x2f, 0x2e, 0x4c, + 0xb5, 0x90, 0xfc, 0x88, 0x52, 0xa9, 0x6a, 0x27, 0x03, 0xf7, 0x44, 0xb8, 0x39, 0xf6, 0x1b, 0xff, + 0x2c, 0x07, 0x95, 0x4f, 0xac, 0xa0, 0x27, 0x97, 0x0e, 0xed, 0x40, 0x2d, 0x74, 0x6e, 0x8c, 0x22, + 0x74, 0x89, 0x85, 0x58, 0x36, 0x46, 0xa6, 0xd2, 0x32, 0x3a, 0x56, 0x7b, 0x2a, 0x81, 0x89, 0xb2, + 0x9c, 0x1e, 0x19, 0x84, 0xa2, 0x72, 0xd9, 0xa2, 0x18, 0xa3, 0x2a, 0x4a, 0x25, 0xa0, 0x03, 0xa8, + 0x8f, 0x3c, 0xf7, 0xd4, 0x23, 0xbe, 0x1f, 0x0a, 0xe3, 0x61, 0x0c, 0xa7, 0x08, 0x3b, 0x14, 0xac, + 0x91, 0xb8, 0x85, 0x91, 0x4e, 0x7a, 0xb4, 0x10, 0xe5, 0x33, 0xdc, 0x39, 0xfd, 0x57, 0x0e, 0x50, + 0x72, 0x52, 0xbf, 0x6b, 0x8a, 0x77, 0x17, 0x6a, 0x7e, 0x60, 0x79, 0x89, 0xcd, 0x56, 0x65, 0xd4, + 0xd0, 0xe3, 0xbf, 0x0d, 0xa1, 0x42, 0xc7, 0x8e, 0x1b, 0xd8, 0x2f, 0x2e, 0x45, 0x96, 0x5c, 0x93, + 0xe4, 0x7d, 0x46, 0x45, 0x1d, 0x28, 0xbc, 0xb0, 0x07, 0x01, 0xf1, 0xfc, 0xc6, 0xdc, 0x5a, 0x7e, + 0xbd, 0xb6, 0xf9, 0xe0, 0x55, 0xcb, 0xb0, 0xf1, 0x21, 0xe3, 0xef, 0x5e, 0x8e, 0x88, 0x29, 0xc7, + 0xaa, 0x99, 0xe7, 0xbc, 0x96, 0x8d, 0xdf, 0x80, 0xe2, 0x4b, 0x2a, 0x82, 0xde, 0xb2, 0x0b, 0x3c, + 0x59, 0x64, 0x6d, 0x7e, 0xc9, 0x7e, 0xe1, 0x59, 0xa7, 0x43, 0xe2, 0x04, 0xf2, 0x1e, 0x28, 0xdb, + 0xf8, 0x2e, 0x40, 0x04, 0x43, 0x5d, 0xfe, 0xfe, 0xc1, 0xe1, 0xb3, 0x6e, 0x7d, 0x06, 0x55, 0xa0, + 0xb8, 0x7f, 0xb0, 0xdd, 0xd9, 0xeb, 0xd0, 0xf8, 0x80, 0x5b, 0xd2, 0xa4, 0xda, 0x5a, 0xaa, 0x98, + 0x86, 0x86, 0x89, 0x57, 0x60, 0x39, 0x6d, 0x01, 0x69, 0x2e, 0x5a, 0x15, 0xbb, 0x74, 0xaa, 0xa3, + 0xa2, 0x42, 0xe7, 0xf4, 0xe9, 0x36, 0xa0, 0xc0, 0x77, 0x6f, 0x5f, 0x24, 0xe7, 0xb2, 0x49, 0x0d, + 0xc1, 0x37, 0x23, 0xe9, 0x8b, 0x55, 0x0a, 0xdb, 0xa9, 0xee, 0x65, 0x2e, 0xd5, 0xbd, 0xa0, 0x3b, + 0x50, 0x0d, 0x4f, 0x83, 0xe5, 0x8b, 0x5c, 0xa0, 0x64, 0x56, 0xe4, 0x46, 0xa7, 0x34, 0xcd, 0xe8, + 0x05, 0xdd, 0xe8, 0xe8, 0x2e, 0xcc, 0x93, 0x09, 0x71, 0x02, 0xbf, 0x51, 0x66, 0x11, 0xa3, 0x2a, + 0x73, 0xf7, 0x0e, 0xa5, 0x9a, 0xa2, 0x13, 0x7f, 0x17, 0x16, 0xd9, 0x1d, 0xe9, 0xb1, 0x67, 0x39, + 0xea, 0x65, 0xae, 0xdb, 0xdd, 0x13, 0xe6, 0xa6, 0x3f, 0x51, 0x0d, 0x72, 0x3b, 0xdb, 0xc2, 0x08, + 0xb9, 0x9d, 0x6d, 0xfc, 0x13, 0x03, 0x90, 0x3a, 0x6e, 0x2a, 0x3b, 0xc7, 0x84, 0x4b, 0xf8, 0x7c, + 0x04, 0xbf, 0x0c, 0x73, 0xc4, 0xf3, 0x5c, 0x8f, 0x59, 0xb4, 0x64, 0xf2, 0x06, 0x7e, 0x4b, 0xe8, + 0x60, 0x92, 0x89, 0x7b, 0x1e, 0x9e, 0x41, 0x2e, 0xcd, 0x08, 0x55, 0xdd, 0x85, 0x25, 0x8d, 0x6b, + 0xaa, 0xc8, 0xf5, 0x21, 0x2c, 0x30, 0x61, 0x5b, 0x67, 0xa4, 0x77, 0x3e, 0x72, 0x6d, 0x27, 0x81, + 0x47, 0x57, 0x2e, 0x72, 0xb0, 0x74, 0x1e, 0x7c, 0x62, 0x95, 0x90, 0xd8, 0xed, 0xee, 0xe1, 0xcf, + 0x60, 0x25, 0x26, 0x47, 0xaa, 0xff, 0x27, 0x50, 0xee, 0x85, 0x44, 0x5f, 0xe4, 0x3a, 0xb7, 0x74, + 0xe5, 0xe2, 0x43, 0xd5, 0x11, 0xf8, 0x00, 0xae, 0x27, 0x44, 0x4f, 0x35, 0xe7, 0xb7, 0xe1, 0x1a, + 0x13, 0xb8, 0x4b, 0xc8, 0xa8, 0x3d, 0xb0, 0x27, 0x99, 0x96, 0x1e, 0x89, 0x49, 0x29, 0x8c, 0x5f, + 0xef, 0xbe, 0xc0, 0x7f, 0x24, 0x10, 0xbb, 0xf6, 0x90, 0x74, 0xdd, 0xbd, 0x6c, 0xdd, 0x68, 0x34, + 0x3b, 0x27, 0x97, 0xbe, 0x48, 0x6b, 0xd8, 0x6f, 0xfc, 0x2f, 0x86, 0x30, 0x95, 0x3a, 0xfc, 0x6b, + 0xde, 0xc9, 0xab, 0x00, 0xa7, 0xf4, 0xc8, 0x90, 0x3e, 0xed, 0xe0, 0x15, 0x15, 0x85, 0x12, 0xea, + 0x49, 0xfd, 0x77, 0x45, 0xe8, 0xb9, 0x2c, 0xf6, 0x39, 0xfb, 0x27, 0xf4, 0x72, 0xb7, 0xa0, 0xcc, + 0x08, 0x47, 0x81, 0x15, 0x8c, 0xfd, 0xc4, 0x62, 0xfc, 0x95, 0xd8, 0xf6, 0x72, 0xd0, 0x54, 0xf3, + 0xfa, 0x16, 0xcc, 0xb3, 0xcb, 0x84, 0x4c, 0xa5, 0x6f, 0xa4, 0xec, 0x47, 0xae, 0x87, 0x29, 0x18, + 0xf1, 0xcf, 0x0c, 0x98, 0x7f, 0xca, 0x4a, 0xb0, 0x8a, 0x6a, 0xb3, 0x72, 0x2d, 0x1c, 0x6b, 0xc8, + 0x0b, 0x43, 0x25, 0x93, 0xfd, 0x66, 0xa9, 0x27, 0x21, 0xde, 0x33, 0x73, 0x8f, 0xa7, 0xb8, 0x25, + 0x33, 0x6c, 0x53, 0x9b, 0xf5, 0x06, 0x36, 0x71, 0x02, 0xd6, 0x3b, 0xcb, 0x7a, 0x15, 0x0a, 0xcd, + 0x9e, 0x6d, 0x7f, 0x8f, 0x58, 0x9e, 0x23, 0x8a, 0xa6, 0x45, 0x33, 0x22, 0xe0, 0x3d, 0xa8, 0x73, + 0x3d, 0xda, 0xfd, 0xbe, 0x92, 0x60, 0x86, 0x68, 0x46, 0x0c, 0x4d, 0x93, 0x96, 0x8b, 0x4b, 0xfb, + 0xa5, 0x01, 0x8b, 0x8a, 0xb8, 0xa9, 0xac, 0xfa, 0x2e, 0xcc, 0xf3, 0x22, 0xb5, 0xc8, 0x74, 0x96, + 0xf5, 0x51, 0x1c, 0xc6, 0x14, 0x3c, 0x68, 0x03, 0x0a, 0xfc, 0x97, 0xbc, 0x03, 0xa4, 0xb3, 0x4b, + 0x26, 0x7c, 0x17, 0x96, 0x04, 0x89, 0x0c, 0xdd, 0xb4, 0x83, 0xc1, 0x16, 0x03, 0xff, 0x05, 0x2c, + 0xeb, 0x6c, 0x53, 0x4d, 0x49, 0x51, 0x32, 0xf7, 0x3a, 0x4a, 0xb6, 0xa5, 0x92, 0xcf, 0x46, 0x7d, + 0x25, 0x8f, 0x8a, 0xef, 0x18, 0x75, 0xbd, 0x72, 0xfa, 0x7a, 0x45, 0x13, 0x90, 0x22, 0xbe, 0xd1, + 0x09, 0x7c, 0x20, 0xb7, 0xc3, 0x9e, 0xed, 0x87, 0x3e, 0x1c, 0x43, 0x65, 0x60, 0x3b, 0xc4, 0xf2, + 0x44, 0xe5, 0xdc, 0xe0, 0x95, 0x73, 0x95, 0x86, 0xbf, 0x00, 0xa4, 0x0e, 0xfc, 0x46, 0x95, 0xbe, + 0x27, 0x4d, 0x76, 0xe8, 0xb9, 0x43, 0x37, 0xd3, 0xec, 0xf8, 0x2f, 0xe1, 0x5a, 0x8c, 0xef, 0x1b, + 0x55, 0x73, 0x09, 0x16, 0xb7, 0x89, 0x4c, 0x68, 0xa4, 0xdb, 0xfb, 0x08, 0x90, 0x4a, 0x9c, 0x2a, + 0xb2, 0xb5, 0x60, 0xf1, 0xa9, 0x3b, 0xa1, 0x2e, 0x92, 0x52, 0x23, 0xdf, 0xc0, 0xeb, 0x10, 0xa1, + 0x29, 0xc2, 0x36, 0x05, 0x57, 0x07, 0x4c, 0x05, 0xfe, 0x1f, 0x06, 0x54, 0xda, 0x03, 0xcb, 0x1b, + 0x4a, 0xe0, 0xef, 0xc1, 0x3c, 0xbf, 0x5d, 0x8b, 0x82, 0xd6, 0x3d, 0x5d, 0x8c, 0xca, 0xcb, 0x1b, + 0x6d, 0x7e, 0x17, 0x17, 0xa3, 0xa8, 0xe2, 0xe2, 0xcd, 0x6b, 0x3b, 0xf6, 0x06, 0xb6, 0x8d, 0xde, + 0x83, 0x39, 0x8b, 0x0e, 0x61, 0xa1, 0xa8, 0x16, 0xaf, 0x6b, 0x30, 0x69, 0xec, 0x0e, 0xc0, 0xb9, + 0xf0, 0x77, 0xa0, 0xac, 0x20, 0xa0, 0x02, 0xe4, 0x1f, 0x77, 0x44, 0xc2, 0xde, 0xde, 0xea, 0xee, + 0x3c, 0xe7, 0x05, 0x9d, 0x1a, 0xc0, 0x76, 0x27, 0x6c, 0xe7, 0xf0, 0xa7, 0x62, 0x94, 0x70, 0xfb, + 0xaa, 0x3e, 0x46, 0x96, 0x3e, 0xb9, 0xd7, 0xd2, 0xe7, 0x02, 0xaa, 0x62, 0xfa, 0xd3, 0x86, 0x31, + 0x26, 0x2f, 0x23, 0x8c, 0x29, 0xca, 0x9b, 0x82, 0x11, 0xff, 0xda, 0x80, 0xfa, 0xb6, 0xfb, 0xd2, + 0x39, 0xf5, 0xac, 0x7e, 0x78, 0x4e, 0x3e, 0x8c, 0xad, 0xd4, 0x46, 0xac, 0x38, 0x1a, 0xe3, 0x8f, + 0x08, 0xb1, 0x15, 0x6b, 0x44, 0x65, 0x43, 0x1e, 0x0b, 0x65, 0x13, 0x7f, 0x00, 0x0b, 0xb1, 0x41, + 0xd4, 0xf6, 0xcf, 0xdb, 0x7b, 0x3b, 0xdb, 0xd4, 0xd6, 0xac, 0xb0, 0xd6, 0xd9, 0x6f, 0x3f, 0xda, + 0xeb, 0x88, 0x07, 0xa4, 0xf6, 0xfe, 0x56, 0x67, 0xaf, 0x9e, 0xc3, 0x3d, 0x58, 0x54, 0xe0, 0xa7, + 0x7d, 0x19, 0xc8, 0xd0, 0x6e, 0x01, 0xaa, 0x22, 0xda, 0x8b, 0x43, 0xf9, 0xef, 0x39, 0xa8, 0x49, + 0xca, 0xd7, 0x83, 0x89, 0x56, 0x60, 0xbe, 0x7f, 0x72, 0x64, 0x7f, 0x21, 0x5f, 0x8e, 0x44, 0x8b, + 0xd2, 0x07, 0x1c, 0x87, 0x3f, 0xdf, 0x8a, 0x16, 0x0d, 0xe3, 0x9e, 0xf5, 0x22, 0xd8, 0x71, 0xfa, + 0xe4, 0x82, 0x25, 0x05, 0xb3, 0x66, 0x44, 0x60, 0x15, 0x26, 0xf1, 0xcc, 0xcb, 0x6e, 0x56, 0xca, + 0xb3, 0x2f, 0xba, 0x0f, 0x75, 0xfa, 0xbb, 0x3d, 0x1a, 0x0d, 0x6c, 0xd2, 0xe7, 0x02, 0x0a, 0x8c, + 0x27, 0x41, 0xa7, 0xe8, 0xec, 0x2e, 0xe2, 0x37, 0x8a, 0x2c, 0x2c, 0x89, 0x16, 0x5a, 0x83, 0x32, + 0xd7, 0x6f, 0xc7, 0x79, 0xe6, 0x13, 0xf6, 0xf6, 0x99, 0x37, 0x55, 0x92, 0x9e, 0x66, 0x40, 0x3c, + 0xcd, 0x58, 0x82, 0xc5, 0xf6, 0x38, 0x38, 0xeb, 0x38, 0x34, 0x56, 0x48, 0x2b, 0x2f, 0x03, 0xa2, + 0xc4, 0x6d, 0xdb, 0x57, 0xa9, 0x82, 0x55, 0x5f, 0x90, 0x0e, 0x2c, 0x51, 0x22, 0x71, 0x02, 0xbb, + 0xa7, 0xc4, 0x55, 0x99, 0x79, 0x19, 0xb1, 0xcc, 0xcb, 0xf2, 0xfd, 0x97, 0xae, 0xd7, 0x17, 0x36, + 0x0f, 0xdb, 0xf8, 0x9f, 0x0c, 0x0e, 0xf9, 0xcc, 0xd7, 0xd2, 0xa7, 0xdf, 0x51, 0x0c, 0x7a, 0x1f, + 0x0a, 0xee, 0x88, 0xbd, 0xf0, 0x8b, 0x32, 0xcc, 0xca, 0x06, 0xff, 0x26, 0x60, 0x43, 0x08, 0x3e, + 0xe0, 0xbd, 0xa6, 0x64, 0x43, 0xf7, 0xa0, 0x76, 0x66, 0xf9, 0x67, 0xa4, 0x7f, 0x28, 0x65, 0xf2, + 0x9b, 0x5f, 0x8c, 0x8a, 0xd7, 0x23, 0xfd, 0x1e, 0x93, 0xe0, 0x0a, 0xfd, 0xf0, 0x03, 0xb8, 0x26, + 0x39, 0xc5, 0xeb, 0xc4, 0x15, 0xcc, 0x2f, 0xe1, 0x96, 0x64, 0xde, 0x3a, 0xb3, 0x9c, 0x53, 0x22, + 0x01, 0x7f, 0x5f, 0x0b, 0x24, 0xe7, 0x93, 0x4f, 0x9d, 0xcf, 0x23, 0x68, 0x84, 0xf3, 0x61, 0x37, + 0x6b, 0x77, 0xa0, 0x2a, 0x3a, 0xf6, 0xc5, 0x79, 0x2a, 0x99, 0xec, 0x37, 0xa5, 0x79, 0xee, 0x20, + 0x4c, 0xa5, 0xe9, 0x6f, 0xbc, 0x05, 0x37, 0xa4, 0x0c, 0x71, 0xe7, 0xd5, 0x85, 0x24, 0x14, 0x4f, + 0x13, 0x22, 0x0c, 0x4b, 0x87, 0x5e, 0xbd, 0xf0, 0x2a, 0xa7, 0xbe, 0x04, 0x4c, 0xa6, 0xa1, 0xc8, + 0xbc, 0xc6, 0x37, 0x25, 0x55, 0x4c, 0xc9, 0x96, 0x24, 0x99, 0x0a, 0x50, 0xc9, 0x62, 0xc1, 0x28, + 0x39, 0xb1, 0x60, 0x09, 0xd1, 0x3f, 0x80, 0xd5, 0x50, 0x09, 0x6a, 0xb7, 0x43, 0xe2, 0x0d, 0x6d, + 0xdf, 0x57, 0xea, 0xde, 0x69, 0x13, 0xbf, 0x07, 0xb3, 0x23, 0x22, 0x82, 0x50, 0x79, 0x13, 0xc9, + 0x4d, 0xa9, 0x0c, 0x66, 0xfd, 0xb8, 0x0f, 0xb7, 0xa5, 0x74, 0x6e, 0xd1, 0x54, 0xf1, 0x71, 0xa5, + 0x64, 0x35, 0x30, 0x97, 0x51, 0x0d, 0xcc, 0xc7, 0xde, 0x62, 0x3e, 0xe2, 0x86, 0x94, 0x67, 0x7e, + 0xaa, 0xe4, 0x62, 0x97, 0xdb, 0x34, 0x74, 0x15, 0x53, 0x09, 0xfb, 0x1b, 0xe1, 0x05, 0xbe, 0x2a, + 0x0f, 0x4f, 0xd8, 0x0c, 0xe5, 0x43, 0x87, 0x6c, 0xd2, 0xac, 0x99, 0x2e, 0x80, 0xa9, 0xd6, 0x42, + 0x67, 0x4d, 0x8d, 0x86, 0x4f, 0x60, 0x59, 0xf7, 0x6b, 0x53, 0xe9, 0xb2, 0x0c, 0x73, 0x81, 0x7b, + 0x4e, 0x64, 0xac, 0xe1, 0x0d, 0x69, 0xbb, 0xd0, 0xe7, 0x4d, 0x65, 0x3b, 0x2b, 0x12, 0xc6, 0x4e, + 0xc7, 0xb4, 0xfa, 0xd2, 0x8d, 0x25, 0xef, 0x40, 0xbc, 0x81, 0xf7, 0x61, 0x25, 0xee, 0xd9, 0xa6, + 0x52, 0xf9, 0x39, 0x3f, 0x4b, 0x69, 0xce, 0x6f, 0x2a, 0xb9, 0x1f, 0x47, 0x7e, 0x49, 0xf1, 0x6d, + 0x53, 0x89, 0x34, 0xa1, 0x99, 0xe6, 0xea, 0xbe, 0x8a, 0xa3, 0x13, 0x7a, 0xbe, 0xa9, 0x84, 0xf9, + 0x91, 0xb0, 0xe9, 0x97, 0x3f, 0x72, 0x57, 0xf9, 0x2b, 0xdd, 0x95, 0x38, 0x24, 0x91, 0x43, 0xfd, + 0x1a, 0x36, 0x9d, 0xc0, 0x88, 0x7c, 0xf9, 0xb4, 0x18, 0x34, 0x9c, 0x85, 0x18, 0xac, 0x21, 0x37, + 0xb6, 0x1a, 0x01, 0xa6, 0x5a, 0x8c, 0x4f, 0x22, 0x37, 0x9e, 0x08, 0x12, 0x53, 0x09, 0xfe, 0x14, + 0xd6, 0xb2, 0xe3, 0xc3, 0x34, 0x92, 0xef, 0xb7, 0xa0, 0x14, 0x5e, 0x86, 0x94, 0xef, 0xcd, 0xca, + 0x50, 0xd8, 0x3f, 0x38, 0x3a, 0x6c, 0x6f, 0x75, 0xf8, 0x07, 0x67, 0x5b, 0x07, 0xa6, 0xf9, 0xec, + 0xb0, 0x5b, 0xcf, 0x6d, 0xfe, 0x26, 0x0f, 0xb9, 0xdd, 0xe7, 0xe8, 0x33, 0x98, 0xe3, 0x5f, 0x5f, + 0x5c, 0xf1, 0xc9, 0x4d, 0xf3, 0xaa, 0x0f, 0x4c, 0xf0, 0xf5, 0x9f, 0xfc, 0xf7, 0x6f, 0x7e, 0x91, + 0x5b, 0xc4, 0x95, 0xd6, 0xe4, 0xdb, 0xad, 0xf3, 0x49, 0x8b, 0x85, 0xa9, 0x87, 0xc6, 0x7d, 0xf4, + 0x31, 0xe4, 0x0f, 0xc7, 0x01, 0xca, 0xfc, 0x14, 0xa7, 0x99, 0xfd, 0xcd, 0x09, 0xbe, 0xc6, 0x84, + 0x2e, 0x60, 0x10, 0x42, 0x47, 0xe3, 0x80, 0x8a, 0xfc, 0x21, 0x94, 0xd5, 0x2f, 0x46, 0x5e, 0xf9, + 0x7d, 0x4e, 0xf3, 0xd5, 0x5f, 0xa3, 0xe0, 0x5b, 0x0c, 0xea, 0x3a, 0x46, 0x02, 0x8a, 0x7f, 0xd3, + 0xa2, 0xce, 0xa2, 0x7b, 0xe1, 0xa0, 0xcc, 0xaf, 0x77, 0x9a, 0xd9, 0x1f, 0xa8, 0x24, 0x66, 0x11, + 0x5c, 0x38, 0x54, 0xe4, 0x9f, 0x89, 0x6f, 0x53, 0x7a, 0x01, 0xba, 0x9d, 0xf2, 0x6d, 0x82, 0xfa, + 0x0a, 0xdf, 0x5c, 0xcb, 0x66, 0x10, 0x20, 0x37, 0x19, 0xc8, 0x0a, 0x5e, 0x14, 0x20, 0xbd, 0x90, + 0xe5, 0xa1, 0x71, 0x7f, 0xb3, 0x07, 0x73, 0xec, 0x85, 0x0b, 0x7d, 0x2e, 0x7f, 0x34, 0x53, 0x9e, + 0xfa, 0x32, 0x16, 0x5a, 0x7b, 0x1b, 0xc3, 0xcb, 0x0c, 0xa8, 0x86, 0x4b, 0x14, 0x88, 0xbd, 0x6f, + 0x3d, 0x34, 0xee, 0xaf, 0x1b, 0xef, 0x1b, 0x9b, 0xbf, 0x9e, 0x83, 0x39, 0x56, 0xda, 0x45, 0xe7, + 0x00, 0xd1, 0x6b, 0x4f, 0x7c, 0x76, 0x89, 0xf7, 0xa3, 0xf8, 0xec, 0x92, 0x0f, 0x45, 0xb8, 0xc9, + 0x40, 0x97, 0xf1, 0x02, 0x05, 0x65, 0x15, 0xe3, 0x16, 0x2b, 0x82, 0x53, 0x3b, 0xfe, 0xad, 0x21, + 0x2a, 0xdb, 0xfc, 0x2c, 0xa1, 0x34, 0x69, 0xda, 0x93, 0x4f, 0x7c, 0x3b, 0xa4, 0x3c, 0xf7, 0xe0, + 0xef, 0x32, 0xc0, 0x16, 0xae, 0x47, 0x80, 0x1e, 0xe3, 0x78, 0x68, 0xdc, 0xff, 0xbc, 0x81, 0x97, + 0x84, 0x95, 0x63, 0x3d, 0xe8, 0x47, 0x50, 0xd3, 0x9f, 0x34, 0xd0, 0x9d, 0x14, 0xac, 0xf8, 0xcb, + 0x48, 0xf3, 0xad, 0xab, 0x99, 0x84, 0x4e, 0xab, 0x4c, 0x27, 0x01, 0xce, 0x91, 0xcf, 0x09, 0x19, + 0x59, 0x94, 0x49, 0xac, 0x01, 0xfa, 0x47, 0x43, 0xbc, 0x38, 0x45, 0x6f, 0x14, 0x28, 0x4d, 0x7a, + 0xe2, 0x05, 0xa4, 0x79, 0xf7, 0x15, 0x5c, 0x42, 0x89, 0x3f, 0x66, 0x4a, 0x7c, 0x80, 0x97, 0x23, + 0x25, 0x02, 0x7b, 0x48, 0x02, 0x57, 0x68, 0xf1, 0xf9, 0x4d, 0x7c, 0x5d, 0x33, 0x8e, 0xd6, 0x1b, + 0x2d, 0x16, 0x7f, 0x67, 0x48, 0x5d, 0x2c, 0xed, 0xdd, 0x22, 0x75, 0xb1, 0xf4, 0x47, 0x8a, 0xb4, + 0xc5, 0xe2, 0xaf, 0x0a, 0x69, 0x8b, 0x15, 0xf6, 0x6c, 0xfe, 0xff, 0x2c, 0x14, 0xb6, 0xf8, 0x37, + 0xe1, 0xc8, 0x85, 0x52, 0x58, 0xa6, 0x47, 0xab, 0x69, 0x75, 0xc6, 0xe8, 0x5a, 0xd3, 0xbc, 0x9d, + 0xd9, 0x2f, 0x14, 0x7a, 0x93, 0x29, 0xf4, 0x06, 0x5e, 0xa1, 0xc8, 0xe2, 0xb3, 0xf3, 0x16, 0x2f, + 0x66, 0xb5, 0xac, 0x7e, 0x9f, 0x1a, 0xe2, 0xcf, 0xa1, 0xa2, 0xd6, 0xd1, 0xd1, 0x9b, 0xa9, 0xb5, + 0x4d, 0xb5, 0x14, 0xdf, 0xc4, 0x57, 0xb1, 0x08, 0xe4, 0xb7, 0x18, 0xf2, 0x2a, 0xbe, 0x91, 0x82, + 0xec, 0x31, 0x56, 0x0d, 0x9c, 0xd7, 0xc0, 0xd3, 0xc1, 0xb5, 0x12, 0x7b, 0x3a, 0xb8, 0x5e, 0x42, + 0xbf, 0x12, 0x7c, 0xcc, 0x58, 0x29, 0xb8, 0x0f, 0x10, 0x55, 0xb2, 0x51, 0xaa, 0x2d, 0x95, 0x7b, + 0x5d, 0xdc, 0x39, 0x24, 0x8b, 0xe0, 0x18, 0x33, 0x58, 0xb1, 0xef, 0x62, 0xb0, 0x03, 0xdb, 0x0f, + 0xf8, 0xc1, 0xac, 0x6a, 0xa5, 0x69, 0x94, 0x3a, 0x1f, 0xbd, 0xbe, 0xdd, 0xbc, 0x73, 0x25, 0x8f, + 0x40, 0xbf, 0xcb, 0xd0, 0x6f, 0xe3, 0x66, 0x0a, 0xfa, 0x88, 0xf3, 0xd2, 0xcd, 0xf6, 0xe3, 0x02, + 0x94, 0x9f, 0x5a, 0xb6, 0x13, 0x10, 0xc7, 0x72, 0x7a, 0x04, 0x9d, 0xc0, 0x1c, 0x8b, 0xd4, 0x71, + 0x47, 0xac, 0x96, 0x6d, 0xe3, 0x8e, 0x58, 0xab, 0x69, 0xe2, 0x35, 0x06, 0xdc, 0xc4, 0xd7, 0x28, + 0xf0, 0x30, 0x12, 0xdd, 0x62, 0xa5, 0x48, 0x3a, 0xe9, 0x17, 0x30, 0x2f, 0x5e, 0xfb, 0x62, 0x82, + 0xb4, 0xe2, 0x4f, 0xf3, 0x66, 0x7a, 0x67, 0xda, 0x5e, 0x56, 0x61, 0x7c, 0xc6, 0x47, 0x71, 0x26, + 0x00, 0x51, 0x8d, 0x3d, 0xbe, 0xa2, 0x89, 0x92, 0x7c, 0x73, 0x2d, 0x9b, 0x21, 0xcd, 0xa6, 0x2a, + 0x66, 0x3f, 0xe4, 0xa5, 0xb8, 0x7f, 0x0a, 0xb3, 0x4f, 0x2c, 0xff, 0x0c, 0xc5, 0x62, 0xaf, 0xf2, + 0xad, 0x58, 0xb3, 0x99, 0xd6, 0x25, 0x50, 0x6e, 0x33, 0x94, 0x1b, 0xdc, 0x95, 0xa9, 0x28, 0x67, + 0x96, 0x7f, 0x26, 0xec, 0xc7, 0x3f, 0x1d, 0x8b, 0xdb, 0x4f, 0xfb, 0xfc, 0x2c, 0x6e, 0x3f, 0xfd, + 0x6b, 0xb3, 0x6c, 0xfb, 0x51, 0x94, 0xf3, 0x09, 0xc5, 0x19, 0x41, 0x51, 0x7e, 0xad, 0x85, 0x62, + 0x4f, 0xf7, 0xb1, 0x2f, 0xbb, 0x9a, 0xab, 0x59, 0xdd, 0x02, 0xed, 0x0e, 0x43, 0xbb, 0x85, 0x1b, + 0x89, 0xd5, 0x12, 0x9c, 0x0f, 0x8d, 0xfb, 0xef, 0x1b, 0xe8, 0x47, 0x00, 0xd1, 0xc3, 0x44, 0xe2, + 0x0c, 0xc6, 0xdf, 0x38, 0x12, 0x67, 0x30, 0xf1, 0xa6, 0x81, 0x37, 0x18, 0xee, 0x3a, 0xbe, 0x13, + 0xc7, 0x0d, 0x3c, 0xcb, 0xf1, 0x5f, 0x10, 0xef, 0x3d, 0x5e, 0x67, 0xf5, 0xcf, 0xec, 0x11, 0x9d, + 0xb2, 0x07, 0xa5, 0xb0, 0xee, 0x1c, 0xf7, 0xb7, 0xf1, 0x7a, 0x78, 0xdc, 0xdf, 0x26, 0x0a, 0xd6, + 0xba, 0xe3, 0xd1, 0xf6, 0x8b, 0x64, 0xa5, 0x47, 0xf0, 0x97, 0x75, 0x98, 0xa5, 0x79, 0x37, 0x4d, + 0x4f, 0xa2, 0xca, 0x49, 0x7c, 0xf6, 0x89, 0x3a, 0x6a, 0x7c, 0xf6, 0xc9, 0xa2, 0x8b, 0x9e, 0x9e, + 0xd0, 0x6b, 0x56, 0x8b, 0x17, 0x29, 0xe8, 0x4c, 0x5d, 0x28, 0x2b, 0xa5, 0x15, 0x94, 0x22, 0x4c, + 0x2f, 0xd0, 0xc6, 0x03, 0x5e, 0x4a, 0x5d, 0x06, 0xbf, 0xc1, 0xf0, 0xae, 0xf1, 0x80, 0xc7, 0xf0, + 0xfa, 0x9c, 0x83, 0x02, 0x8a, 0xd9, 0x89, 0x93, 0x9f, 0x32, 0x3b, 0xfd, 0xf4, 0xaf, 0x65, 0x33, + 0x64, 0xce, 0x2e, 0x3a, 0xfa, 0x2f, 0xa1, 0xa2, 0x16, 0x58, 0x50, 0x8a, 0xf2, 0xb1, 0xa2, 0x72, + 0x3c, 0x92, 0xa4, 0xd5, 0x67, 0x74, 0xdf, 0xc6, 0x20, 0x2d, 0x85, 0x8d, 0x02, 0x0f, 0xa0, 0x20, + 0x2a, 0x2e, 0x69, 0x26, 0xd5, 0x0b, 0xd0, 0x69, 0x26, 0x8d, 0x95, 0x6b, 0xf4, 0xfc, 0x99, 0x21, + 0xd2, 0x4b, 0xa5, 0x8c, 0xd6, 0x02, 0xed, 0x31, 0x09, 0xb2, 0xd0, 0xa2, 0x5a, 0x66, 0x16, 0x9a, + 0x72, 0xa1, 0xcf, 0x42, 0x3b, 0x25, 0x81, 0xf0, 0x07, 0xf2, 0xa2, 0x8c, 0x32, 0x84, 0xa9, 0x11, + 0x12, 0x5f, 0xc5, 0x92, 0x76, 0xbd, 0x89, 0x00, 0x65, 0x78, 0xbc, 0x00, 0x88, 0xea, 0x41, 0xf1, + 0x9c, 0x35, 0xb5, 0x0e, 0x1e, 0xcf, 0x59, 0xd3, 0x4b, 0x4a, 0xba, 0x8f, 0x8d, 0x70, 0xf9, 0xed, + 0x8a, 0x22, 0xff, 0xdc, 0x00, 0x94, 0x2c, 0x1d, 0xa1, 0x07, 0xe9, 0xd2, 0x53, 0xab, 0xeb, 0xcd, + 0x77, 0x5f, 0x8f, 0x39, 0xcd, 0x21, 0x47, 0x2a, 0xf5, 0x18, 0xf7, 0xe8, 0x25, 0x55, 0xea, 0xc7, + 0x06, 0x54, 0xb5, 0xba, 0x13, 0xba, 0x97, 0xb1, 0xa6, 0xb1, 0xa2, 0x7b, 0xf3, 0xed, 0x57, 0xf2, + 0xa5, 0x25, 0xf3, 0xca, 0x0e, 0x90, 0xb7, 0x9a, 0x9f, 0x1a, 0x50, 0xd3, 0xeb, 0x54, 0x28, 0x43, + 0x76, 0xa2, 0x68, 0xdf, 0x5c, 0x7f, 0x35, 0xe3, 0xd5, 0xcb, 0x13, 0x5d, 0x68, 0x06, 0x50, 0x10, + 0x95, 0xad, 0xb4, 0x8d, 0xaf, 0x97, 0xfb, 0xd3, 0x36, 0x7e, 0xac, 0x2c, 0x96, 0xb2, 0xf1, 0x3d, + 0x77, 0x40, 0x94, 0x63, 0x26, 0x4a, 0x5f, 0x59, 0x68, 0x57, 0x1f, 0xb3, 0x58, 0xdd, 0x2c, 0x0b, + 0x2d, 0x3a, 0x66, 0xb2, 0xe6, 0x85, 0x32, 0x84, 0xbd, 0xe2, 0x98, 0xc5, 0x4b, 0x66, 0x29, 0xc7, + 0x8c, 0x01, 0x2a, 0xc7, 0x2c, 0xaa, 0x4e, 0xa5, 0x1d, 0xb3, 0xc4, 0xeb, 0x45, 0xda, 0x31, 0x4b, + 0x16, 0xb8, 0x52, 0xd6, 0x91, 0xe1, 0x6a, 0xc7, 0x6c, 0x29, 0xa5, 0x90, 0x85, 0xde, 0xcd, 0x30, + 0x62, 0xea, 0xa3, 0x48, 0xf3, 0xbd, 0xd7, 0xe4, 0xce, 0xdc, 0xe3, 0xdc, 0xfc, 0x72, 0x8f, 0xff, + 0x9d, 0x01, 0xcb, 0x69, 0x45, 0x30, 0x94, 0x81, 0x93, 0xf1, 0x98, 0xd2, 0xdc, 0x78, 0x5d, 0xf6, + 0xab, 0xad, 0x15, 0xee, 0xfa, 0x47, 0xf5, 0x7f, 0xfb, 0x72, 0xd5, 0xf8, 0xcf, 0x2f, 0x57, 0x8d, + 0xff, 0xf9, 0x72, 0xd5, 0xf8, 0xfb, 0xff, 0x5d, 0x9d, 0x39, 0x99, 0x67, 0xff, 0xd5, 0xf8, 0xdb, + 0xbf, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x8b, 0x54, 0x11, 0xdf, 0xef, 0x3c, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/api/etcdserverpb/rpc.proto b/api/etcdserverpb/rpc.proto index 14391378ada9..4ccc234744c2 100644 --- a/api/etcdserverpb/rpc.proto +++ b/api/etcdserverpb/rpc.proto @@ -217,7 +217,7 @@ service Maintenance { // It only iterates "key" bucket in backend storage. rpc HashKV(HashKVRequest) returns (HashKVResponse) { option (google.api.http) = { - post: "/v3/maintenance/hash" + post: "/v3/maintenance/hashkv" body: "*" }; } diff --git a/api/go.mod b/api/go.mod index b14057b44191..43d649a758bd 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,14 +1,26 @@ module go.etcd.io/etcd/api/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 require ( github.com/coreos/go-semver v0.3.0 github.com/gogo/protobuf v1.3.2 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.4 github.com/grpc-ecosystem/grpc-gateway v1.16.0 - google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 - google.golang.org/grpc v1.37.0 + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d + google.golang.org/grpc v1.59.0 +) + +require ( + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) // Bad imports are sometimes causing attempts to pull that code. diff --git a/api/go.sum b/api/go.sum index 52d9b2c3ed3b..229c3cb36bb7 100644 --- a/api/go.sum +++ b/api/go.sum @@ -5,14 +5,11 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -22,33 +19,19 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -69,8 +52,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -83,11 +67,13 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -99,37 +85,30 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/api/v3rpc/rpctypes/error.go b/api/v3rpc/rpctypes/error.go index c03e91cfbc82..23201302e833 100644 --- a/api/v3rpc/rpctypes/error.go +++ b/api/v3rpc/rpctypes/error.go @@ -45,6 +45,7 @@ var ( ErrGRPCMemberNotLearner = status.New(codes.FailedPrecondition, "etcdserver: can only promote a learner member").Err() ErrGRPCLearnerNotReady = status.New(codes.FailedPrecondition, "etcdserver: can only promote a learner member which is in sync with leader").Err() ErrGRPCTooManyLearners = status.New(codes.FailedPrecondition, "etcdserver: too many learner members in cluster").Err() + ErrGRPCClusterIdMismatch = status.New(codes.FailedPrecondition, "etcdserver: cluster ID mismatch").Err() ErrGRPCRequestTooLarge = status.New(codes.InvalidArgument, "etcdserver: request is too large").Err() ErrGRPCRequestTooManyRequests = status.New(codes.ResourceExhausted, "etcdserver: too many requests").Err() @@ -58,12 +59,14 @@ var ( ErrGRPCRoleNotFound = status.New(codes.FailedPrecondition, "etcdserver: role name not found").Err() ErrGRPCRoleEmpty = status.New(codes.InvalidArgument, "etcdserver: role name is empty").Err() ErrGRPCAuthFailed = status.New(codes.InvalidArgument, "etcdserver: authentication failed, invalid user ID or password").Err() + ErrGRPCPermissionNotGiven = status.New(codes.InvalidArgument, "etcdserver: permission not given").Err() ErrGRPCPermissionDenied = status.New(codes.PermissionDenied, "etcdserver: permission denied").Err() ErrGRPCRoleNotGranted = status.New(codes.FailedPrecondition, "etcdserver: role is not granted to the user").Err() ErrGRPCPermissionNotGranted = status.New(codes.FailedPrecondition, "etcdserver: permission is not granted to the role").Err() ErrGRPCAuthNotEnabled = status.New(codes.FailedPrecondition, "etcdserver: authentication is not enabled").Err() ErrGRPCInvalidAuthToken = status.New(codes.Unauthenticated, "etcdserver: invalid auth token").Err() ErrGRPCInvalidAuthMgmt = status.New(codes.InvalidArgument, "etcdserver: invalid auth management").Err() + ErrGRPCAuthOldRevision = status.New(codes.InvalidArgument, "etcdserver: revision of auth store is old").Err() ErrGRPCNoLeader = status.New(codes.Unavailable, "etcdserver: no leader").Err() ErrGRPCNotLeader = status.New(codes.FailedPrecondition, "etcdserver: not leader").Err() @@ -73,6 +76,7 @@ var ( ErrGRPCTimeout = status.New(codes.Unavailable, "etcdserver: request timed out").Err() ErrGRPCTimeoutDueToLeaderFail = status.New(codes.Unavailable, "etcdserver: request timed out, possibly due to previous leader failure").Err() ErrGRPCTimeoutDueToConnectionLost = status.New(codes.Unavailable, "etcdserver: request timed out, possibly due to connection lost").Err() + ErrGRPCTimeoutWaitAppliedIndex = status.New(codes.Unavailable, "etcdserver: request timed out, waiting for the applied index took too long").Err() ErrGRPCUnhealthy = status.New(codes.Unavailable, "etcdserver: unhealthy cluster").Err() ErrGRPCCorrupt = status.New(codes.DataLoss, "etcdserver: corrupt cluster").Err() ErrGPRCNotSupportedForLearner = status.New(codes.Unavailable, "etcdserver: rpc not supported for learner").Err() @@ -111,6 +115,7 @@ var ( ErrorDesc(ErrGRPCMemberNotLearner): ErrGRPCMemberNotLearner, ErrorDesc(ErrGRPCLearnerNotReady): ErrGRPCLearnerNotReady, ErrorDesc(ErrGRPCTooManyLearners): ErrGRPCTooManyLearners, + ErrorDesc(ErrGRPCClusterIdMismatch): ErrGRPCClusterIdMismatch, ErrorDesc(ErrGRPCRequestTooLarge): ErrGRPCRequestTooLarge, ErrorDesc(ErrGRPCRequestTooManyRequests): ErrGRPCRequestTooManyRequests, @@ -130,6 +135,7 @@ var ( ErrorDesc(ErrGRPCAuthNotEnabled): ErrGRPCAuthNotEnabled, ErrorDesc(ErrGRPCInvalidAuthToken): ErrGRPCInvalidAuthToken, ErrorDesc(ErrGRPCInvalidAuthMgmt): ErrGRPCInvalidAuthMgmt, + ErrorDesc(ErrGRPCAuthOldRevision): ErrGRPCAuthOldRevision, ErrorDesc(ErrGRPCNoLeader): ErrGRPCNoLeader, ErrorDesc(ErrGRPCNotLeader): ErrGRPCNotLeader, @@ -194,7 +200,9 @@ var ( ErrPermissionNotGranted = Error(ErrGRPCPermissionNotGranted) ErrAuthNotEnabled = Error(ErrGRPCAuthNotEnabled) ErrInvalidAuthToken = Error(ErrGRPCInvalidAuthToken) + ErrAuthOldRevision = Error(ErrGRPCAuthOldRevision) ErrInvalidAuthMgmt = Error(ErrGRPCInvalidAuthMgmt) + ErrClusterIdMismatch = Error(ErrGRPCClusterIdMismatch) ErrNoLeader = Error(ErrGRPCNoLeader) ErrNotLeader = Error(ErrGRPCNotLeader) @@ -204,6 +212,7 @@ var ( ErrTimeout = Error(ErrGRPCTimeout) ErrTimeoutDueToLeaderFail = Error(ErrGRPCTimeoutDueToLeaderFail) ErrTimeoutDueToConnectionLost = Error(ErrGRPCTimeoutDueToConnectionLost) + ErrTimeoutWaitAppliedIndex = Error(ErrGRPCTimeoutWaitAppliedIndex) ErrUnhealthy = Error(ErrGRPCUnhealthy) ErrCorrupt = Error(ErrGRPCCorrupt) ErrBadLeaderTransferee = Error(ErrGRPCBadLeaderTransferee) diff --git a/api/version/version.go b/api/version/version.go index af132f821d59..03449b523b60 100644 --- a/api/version/version.go +++ b/api/version/version.go @@ -26,7 +26,7 @@ import ( var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.5.0-alpha.0" + Version = "3.5.21" APIVersion = "unknown" // Git SHA Value will be set during build diff --git a/bill-of-materials.json b/bill-of-materials.json index 7317583b4d15..10dd58041734 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -18,10 +18,10 @@ ] }, { - "project": "github.com/certifi/gocertifi", + "project": "github.com/cenkalti/backoff/v4", "licenses": [ { - "type": "Mozilla Public License 2.0", + "type": "MIT License", "confidence": 1 } ] @@ -44,24 +44,6 @@ } ] }, - { - "project": "github.com/cockroachdb/errors", - "licenses": [ - { - "type": "Apache License 2.0", - "confidence": 1 - } - ] - }, - { - "project": "github.com/cockroachdb/logtags", - "licenses": [ - { - "type": "Apache License 2.0", - "confidence": 1 - } - ] - }, { "project": "github.com/coreos/go-semver/semver", "licenses": [ @@ -117,7 +99,7 @@ ] }, { - "project": "github.com/etcd-io/gofail/runtime", + "project": "github.com/go-logr/logr", "licenses": [ { "type": "Apache License 2.0", @@ -126,29 +108,29 @@ ] }, { - "project": "github.com/form3tech-oss/jwt-go", + "project": "github.com/go-logr/stdr", "licenses": [ { - "type": "MIT License", - "confidence": 0.9891304347826086 + "type": "Apache License 2.0", + "confidence": 1 } ] }, { - "project": "github.com/getsentry/raven-go", + "project": "github.com/gogo/protobuf", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.9663865546218487 + "confidence": 0.9163346613545816 } ] }, { - "project": "github.com/gogo/protobuf", + "project": "github.com/golang-jwt/jwt/v4", "licenses": [ { - "type": "BSD 3-clause \"New\" or \"Revised\" License", - "confidence": 0.9163346613545816 + "type": "MIT License", + "confidence": 0.9891304347826086 } ] }, @@ -215,6 +197,15 @@ } ] }, + { + "project": "github.com/grpc-ecosystem/grpc-gateway/v2", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.979253112033195 + } + ] + }, { "project": "github.com/inconshreveable/mousetrap", "licenses": [ @@ -287,15 +278,6 @@ } ] }, - { - "project": "github.com/pkg/errors", - "licenses": [ - { - "type": "BSD 2-clause \"Simplified\" License", - "confidence": 1 - } - ] - }, { "project": "github.com/pmezard/go-difflib/difflib", "licenses": [ @@ -540,7 +522,7 @@ ] }, { - "project": "go.opentelemetry.io/contrib", + "project": "go.etcd.io/gofail/runtime", "licenses": [ { "type": "Apache License 2.0", @@ -567,16 +549,7 @@ ] }, { - "project": "go.opentelemetry.io/otel/exporters/otlp", - "licenses": [ - { - "type": "Apache License 2.0", - "confidence": 1 - } - ] - }, - { - "project": "go.opentelemetry.io/otel/metric", + "project": "go.opentelemetry.io/otel/exporters/otlp/otlptrace", "licenses": [ { "type": "Apache License 2.0", @@ -585,7 +558,7 @@ ] }, { - "project": "go.opentelemetry.io/otel/sdk", + "project": "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc", "licenses": [ { "type": "Apache License 2.0", @@ -594,7 +567,7 @@ ] }, { - "project": "go.opentelemetry.io/otel/sdk/export/metric", + "project": "go.opentelemetry.io/otel/metric", "licenses": [ { "type": "Apache License 2.0", @@ -603,7 +576,7 @@ ] }, { - "project": "go.opentelemetry.io/otel/sdk/metric", + "project": "go.opentelemetry.io/otel/sdk", "licenses": [ { "type": "Apache License 2.0", @@ -675,7 +648,16 @@ ] }, { - "project": "golang.org/x/sys", + "project": "golang.org/x/sync/errgroup", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, + { + "project": "golang.org/x/sys/unix", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", @@ -702,7 +684,25 @@ ] }, { - "project": "google.golang.org/genproto", + "project": "google.golang.org/genproto/googleapis/api", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "google.golang.org/genproto/googleapis/rpc", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "google.golang.org/genproto/protobuf/field_mask", "licenses": [ { "type": "Apache License 2.0", diff --git a/build b/build index 60aa15d768d4..7d20f00aa2f0 100755 --- a/build +++ b/build @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + echo -e "\\e[91mDEPRECATED!!! Use build.sh script instead.\\e[0m\\n" sleep 1 diff --git a/build.sh b/build.sh index 7a4f8670e6f6..7d12a8c7e340 100755 --- a/build.sh +++ b/build.sh @@ -1,24 +1,50 @@ #!/usr/bin/env bash +set -euo pipefail + +GO_BUILD_FLAGS="${GO_BUILD_FLAGS} ${GOFLAGS:+-$GOFLAGS}" + source ./scripts/test_lib.sh GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") -if [[ -n "$FAILPOINTS" ]]; then +if [[ -n "${FAILPOINTS:-}" ]]; then GIT_SHA="$GIT_SHA"-FAILPOINTS fi VERSION_SYMBOL="${ROOT_MODULE}/api/v3/version.GitSHA" +# use go env if noset +GOOS=${GOOS:-$(go env GOOS)} +GOARCH=${GOARCH:-$(go env GOARCH)} + +CGO_ENABLED="${CGO_ENABLED:-0}" + # Set GO_LDFLAGS="-s" for building without symbols for debugging. # shellcheck disable=SC2206 -GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}") -GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}") +GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}") +GO_BUILD_ENV=("CGO_ENABLED=${CGO_ENABLED}" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-}" "GOOS=${GOOS}" "GOARCH=${GOARCH}") +GOFAIL_VERSION=$(cd tools/mod && go list -m -f '{{.Version}}' go.etcd.io/gofail) # enable/disable failpoints toggle_failpoints() { mode="$1" if command -v gofail >/dev/null 2>&1; then - run gofail "$mode" server/etcdserver/ server/mvcc/backend/ + run gofail "$mode" server/etcdserver/ server/lease/leasehttp server/mvcc/ server/wal/ server/mvcc/backend/ + if [[ "$mode" == "enable" ]]; then + go get go.etcd.io/gofail@"${GOFAIL_VERSION}" + cd ./server && go get go.etcd.io/gofail@"${GOFAIL_VERSION}" + cd ../etcdutl && go get go.etcd.io/gofail@"${GOFAIL_VERSION}" + cd ../etcdctl && go get go.etcd.io/gofail@"${GOFAIL_VERSION}" + cd ../tests && go get go.etcd.io/gofail@"${GOFAIL_VERSION}" + cd ../ + else + go mod tidy + cd ./server && go mod tidy + cd ../etcdutl && go mod tidy + cd ../etcdctl && go mod tidy + cd ../tests && go mod tidy + cd ../ + fi elif [[ "$mode" != "disable" ]]; then log_error "FAILPOINTS set but gofail not found" exit 1 @@ -27,13 +53,13 @@ toggle_failpoints() { toggle_failpoints_default() { mode="disable" - if [[ -n "$FAILPOINTS" ]]; then mode="enable"; fi + if [[ -n "${FAILPOINTS:-}" ]]; then mode="enable"; fi toggle_failpoints "$mode" } etcd_build() { out="bin" - if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi + if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi toggle_failpoints_default run rm -f "${out}/etcd" @@ -41,7 +67,8 @@ etcd_build() { cd ./server # Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK # shellcheck disable=SC2086 - run env "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ + run env "${GO_BUILD_ENV[@]}" go build ${GO_BUILD_FLAGS:-} \ + -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -o="../${out}/etcd" . || return 2 @@ -51,7 +78,8 @@ etcd_build() { # shellcheck disable=SC2086 ( cd ./etcdutl - run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ + run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS:-}" "${GO_BUILD_ENV[@]}" go build ${GO_BUILD_FLAGS:-} \ + -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -o="../${out}/etcdutl" . || return 2 @@ -61,11 +89,23 @@ etcd_build() { # shellcheck disable=SC2086 ( cd ./etcdctl - run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ + run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS:-}" "${GO_BUILD_ENV[@]}" go build ${GO_BUILD_FLAGS:-} \ + -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -o="../${out}/etcdctl" . || return 2 ) || return 2 + + run rm -f "${out}/discover-etcd-initial-cluster" + # shellcheck disable=SC2086 + ( + cd ./openshift-tools/discover-etcd-initial-cluster + run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ + -installsuffix=cgo \ + "-ldflags=${GO_LDFLAGS[*]}" \ + -o="../../${out}/discover-etcd-initial-cluster" . || return 2 + ) || return 2 + # Verify whether symbol we overriden exists # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}" @@ -81,7 +121,7 @@ etcd_build() { tools_build() { out="bin" - if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi + if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi tools_path="tools/benchmark tools/etcd-dump-db tools/etcd-dump-logs @@ -91,9 +131,10 @@ tools_build() { echo "Building" "'${tool}'"... run rm -f "${out}/${tool}" # shellcheck disable=SC2086 - run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} \ + run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS:-}" CGO_ENABLED=${CGO_ENABLED} go build ${GO_BUILD_FLAGS:-} \ + -trimpath \ -installsuffix=cgo \ - "-ldflags='${GO_LDFLAGS[*]}'" \ + "-ldflags=${GO_LDFLAGS[*]}" \ -o="${out}/${tool}" "./${tool}" || return 2 done tests_build "${@}" @@ -101,7 +142,7 @@ tools_build() { tests_build() { out="bin" - if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi + if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi tools_path=" functional/cmd/etcd-agent functional/cmd/etcd-proxy @@ -114,9 +155,9 @@ tests_build() { run rm -f "../${out}/${tool}" # shellcheck disable=SC2086 - run env CGO_ENABLED=0 GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" go build ${GO_BUILD_FLAGS} \ + run env CGO_ENABLED=${CGO_ENABLED} GO_BUILD_FLAGS="${GO_BUILD_FLAGS:-}" go build ${GO_BUILD_FLAGS:-} \ -installsuffix=cgo \ - "-ldflags='${GO_LDFLAGS[*]}'" \ + "-ldflags=${GO_LDFLAGS[*]}" \ -o="../${out}/${tool}" "./${tool}" || return 2 done ) || return 2 diff --git a/client/pkg/fileutil/filereader.go b/client/pkg/fileutil/filereader.go new file mode 100644 index 000000000000..55248888c603 --- /dev/null +++ b/client/pkg/fileutil/filereader.go @@ -0,0 +1,60 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "bufio" + "io" + "io/fs" + "os" +) + +// FileReader is a wrapper of io.Reader. It also provides file info. +type FileReader interface { + io.Reader + FileInfo() (fs.FileInfo, error) +} + +type fileReader struct { + *os.File +} + +func NewFileReader(f *os.File) FileReader { + return &fileReader{f} +} + +func (fr *fileReader) FileInfo() (fs.FileInfo, error) { + return fr.Stat() +} + +// FileBufReader is a wrapper of bufio.Reader. It also provides file info. +type FileBufReader struct { + *bufio.Reader + fi fs.FileInfo +} + +func NewFileBufReader(fr FileReader) *FileBufReader { + bufReader := bufio.NewReader(fr) + fi, err := fr.FileInfo() + if err != nil { + // This should never happen. + panic(err) + } + return &FileBufReader{bufReader, fi} +} + +func (fbr *FileBufReader) FileInfo() fs.FileInfo { + return fbr.fi +} diff --git a/client/pkg/fileutil/filereader_test.go b/client/pkg/fileutil/filereader_test.go new file mode 100644 index 000000000000..2f863cdcef5a --- /dev/null +++ b/client/pkg/fileutil/filereader_test.go @@ -0,0 +1,44 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFileBufReader(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "wal") + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + fi, err := f.Stat() + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + fbr := NewFileBufReader(NewFileReader(f)) + + if !strings.HasPrefix(fbr.FileInfo().Name(), "wal") { + t.Errorf("Unexpected file name: %s", fbr.FileInfo().Name()) + } + assert.Equal(t, fi.Size(), fbr.FileInfo().Size()) + assert.Equal(t, fi.IsDir(), fbr.FileInfo().IsDir()) + assert.Equal(t, fi.Mode(), fbr.FileInfo().Mode()) + assert.Equal(t, fi.ModTime(), fbr.FileInfo().ModTime()) +} diff --git a/client/pkg/fileutil/fileutil.go b/client/pkg/fileutil/fileutil.go index e442c3c92e83..d31ece3e24ea 100644 --- a/client/pkg/fileutil/fileutil.go +++ b/client/pkg/fileutil/fileutil.go @@ -44,16 +44,12 @@ func IsDirWriteable(dir string) error { // TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory // does not exists. TouchDirAll also ensures the given directory is writable. -func TouchDirAll(dir string) error { +func TouchDirAll(lg *zap.Logger, dir string) error { // If path is already a directory, MkdirAll does nothing and returns nil, so, // first check if dir exist with an expected permission mode. if Exist(dir) { err := CheckDirPermission(dir, PrivateDirMode) if err != nil { - lg, _ := zap.NewProduction() - if lg == nil { - lg = zap.NewExample() - } lg.Warn("check file permission", zap.Error(err)) } } else { @@ -70,8 +66,8 @@ func TouchDirAll(dir string) error { // CreateDirAll is similar to TouchDirAll but returns error // if the deepest directory was not empty. -func CreateDirAll(dir string) error { - err := TouchDirAll(dir) +func CreateDirAll(lg *zap.Logger, dir string) error { + err := TouchDirAll(lg, dir) if err == nil { var ns []string ns, err = ReadDir(dir) diff --git a/client/pkg/fileutil/fileutil_test.go b/client/pkg/fileutil/fileutil_test.go index 3a761ff9a7f3..1804163823ff 100644 --- a/client/pkg/fileutil/fileutil_test.go +++ b/client/pkg/fileutil/fileutil_test.go @@ -67,7 +67,7 @@ func TestCreateDirAll(t *testing.T) { defer os.RemoveAll(tmpdir) tmpdir2 := filepath.Join(tmpdir, "testdir") - if err = CreateDirAll(tmpdir2); err != nil { + if err = CreateDirAll(zaptest.NewLogger(t), tmpdir2); err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestCreateDirAll(t *testing.T) { t.Fatal(err) } - if err = CreateDirAll(tmpdir2); err == nil || !strings.Contains(err.Error(), "to be empty, got") { + if err = CreateDirAll(zaptest.NewLogger(t), tmpdir2); err == nil || !strings.Contains(err.Error(), "to be empty, got") { t.Fatalf("unexpected error %v", err) } } @@ -186,7 +186,7 @@ func TestDirPermission(t *testing.T) { tmpdir2 := filepath.Join(tmpdir, "testpermission") // create a new dir with 0700 - if err = CreateDirAll(tmpdir2); err != nil { + if err = CreateDirAll(zaptest.NewLogger(t), tmpdir2); err != nil { t.Fatal(err) } // check dir permission with mode different than created dir diff --git a/client/pkg/fileutil/purge.go b/client/pkg/fileutil/purge.go index e8ac0ca6f58a..b314e068fea1 100644 --- a/client/pkg/fileutil/purge.go +++ b/client/pkg/fileutil/purge.go @@ -25,22 +25,34 @@ import ( ) func PurgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error { - return purgeFile(lg, dirname, suffix, max, interval, stop, nil, nil) + return purgeFile(lg, dirname, suffix, max, interval, stop, nil, nil, true) } func PurgeFileWithDoneNotify(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) { doneC := make(chan struct{}) - errC := purgeFile(lg, dirname, suffix, max, interval, stop, nil, doneC) + errC := purgeFile(lg, dirname, suffix, max, interval, stop, nil, doneC, true) + return doneC, errC +} + +func PurgeFileWithoutFlock(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) { + doneC := make(chan struct{}) + errC := purgeFile(lg, dirname, suffix, max, interval, stop, nil, doneC, false) return doneC, errC } // purgeFile is the internal implementation for PurgeFile which can post purged files to purgec if non-nil. // if donec is non-nil, the function closes it to notify its exit. -func purgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string, donec chan<- struct{}) <-chan error { +func purgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string, donec chan<- struct{}, flock bool) <-chan error { if lg == nil { lg = zap.NewNop() } errC := make(chan error, 1) + lg.Info("started to purge file", + zap.String("dir", dirname), + zap.String("suffix", suffix), + zap.Uint("max", max), + zap.Duration("interval", interval)) + go func() { if donec != nil { defer close(donec) @@ -61,18 +73,25 @@ func purgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval fnames = newfnames for len(newfnames) > int(max) { f := filepath.Join(dirname, newfnames[0]) - l, err := TryLockFile(f, os.O_WRONLY, PrivateFileMode) - if err != nil { - break + var l *LockedFile + if flock { + l, err = TryLockFile(f, os.O_WRONLY, PrivateFileMode) + if err != nil { + lg.Warn("failed to lock file", zap.String("path", f), zap.Error(err)) + break + } } if err = os.Remove(f); err != nil { + lg.Error("failed to remove file", zap.String("path", f), zap.Error(err)) errC <- err return } - if err = l.Close(); err != nil { - lg.Warn("failed to unlock/close", zap.String("path", l.Name()), zap.Error(err)) - errC <- err - return + if flock { + if err = l.Close(); err != nil { + lg.Error("failed to unlock/close", zap.String("path", l.Name()), zap.Error(err)) + errC <- err + return + } } lg.Info("purged", zap.String("path", f)) newfnames = newfnames[1:] diff --git a/client/pkg/fileutil/purge_test.go b/client/pkg/fileutil/purge_test.go index f0e7d6b566f9..a29a8f2e9606 100644 --- a/client/pkg/fileutil/purge_test.go +++ b/client/pkg/fileutil/purge_test.go @@ -45,7 +45,7 @@ func TestPurgeFile(t *testing.T) { stop, purgec := make(chan struct{}), make(chan string, 10) // keep 3 most recent files - errch := purgeFile(zap.NewExample(), dir, "test", 3, time.Millisecond, stop, purgec, nil) + errch := purgeFile(zap.NewExample(), dir, "test", 3, time.Millisecond, stop, purgec, nil, false) select { case f := <-purgec: t.Errorf("unexpected purge on %q", f) @@ -116,7 +116,7 @@ func TestPurgeFileHoldingLockFile(t *testing.T) { } stop, purgec := make(chan struct{}), make(chan string, 10) - errch := purgeFile(zap.NewExample(), dir, "test", 3, time.Millisecond, stop, purgec, nil) + errch := purgeFile(zap.NewExample(), dir, "test", 3, time.Millisecond, stop, purgec, nil, true) for i := 0; i < 5; i++ { select { diff --git a/client/pkg/go.mod b/client/pkg/go.mod index 4cbe1ceb6264..f94c07303102 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -1,10 +1,20 @@ module go.etcd.io/etcd/client/pkg/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 + +require ( + github.com/coreos/go-systemd/v22 v22.3.2 + github.com/stretchr/testify v1.9.0 + go.uber.org/zap v1.17.0 + golang.org/x/sys v0.31.0 +) require ( - github.com/coreos/go-systemd/v22 v22.3.1 - go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 - golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 - google.golang.org/grpc v1.37.0 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/client/pkg/go.sum b/client/pkg/go.sum index 14c70b3330f5..278561afaac5 100644 --- a/client/pkg/go.sum +++ b/client/pkg/go.sum @@ -1,96 +1,31 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/coreos/go-systemd/v22 v22.3.1 h1:7OO2CXWMYNDdaAzP51t4lCCZWwpQHmvPbm9sxWjm3So= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 h1:040c3dLNhgFQkoojH2AMpHCy4SrvhmxdU72d9GLGGE0= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/pkg/logutil/zap.go b/client/pkg/logutil/zap.go index 8fc6e03b77bd..f0f3739aad81 100644 --- a/client/pkg/logutil/zap.go +++ b/client/pkg/logutil/zap.go @@ -16,11 +16,23 @@ package logutil import ( "sort" + "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) +// CreateDefaultZapLogger creates a logger with default zap configuration +func CreateDefaultZapLogger(level zapcore.Level) (*zap.Logger, error) { + lcfg := DefaultZapLoggerConfig + lcfg.Level = zap.NewAtomicLevelAt(level) + c, err := lcfg.Build() + if err != nil { + return nil, err + } + return c, nil +} + // DefaultZapLoggerConfig defines default zap logger configuration. var DefaultZapLoggerConfig = zap.Config{ Level: zap.NewAtomicLevelAt(ConvertToZapLevel(DefaultLogLevel)), @@ -35,15 +47,20 @@ var DefaultZapLoggerConfig = zap.Config{ // copied from "zap.NewProductionEncoderConfig" with some updates EncoderConfig: zapcore.EncoderConfig{ - TimeKey: "ts", - LevelKey: "level", - NameKey: "logger", - CallerKey: "caller", - MessageKey: "msg", - StacktraceKey: "stacktrace", - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, + TimeKey: "ts", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + + // Custom EncodeTime function to ensure we match format and precision of historic capnslog timestamps + EncodeTime: func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(t.Format("2006-01-02T15:04:05.000000Z0700")) + }, + EncodeDuration: zapcore.StringDurationEncoder, EncodeCaller: zapcore.ShortCallerEncoder, }, diff --git a/client/pkg/logutil/zap_grpc.go b/client/pkg/logutil/zap_grpc.go deleted file mode 100644 index fb5cb4e4bc17..000000000000 --- a/client/pkg/logutil/zap_grpc.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logutil - -import ( - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "google.golang.org/grpc/grpclog" -) - -// NewGRPCLoggerV2 converts "*zap.Logger" to "grpclog.LoggerV2". -// It discards all INFO level logging in gRPC, if debug level -// is not enabled in "*zap.Logger". -func NewGRPCLoggerV2(lcfg zap.Config) (grpclog.LoggerV2, error) { - lg, err := lcfg.Build(zap.AddCallerSkip(1)) // to annotate caller outside of "logutil" - if err != nil { - return nil, err - } - return &zapGRPCLogger{lg: lg, sugar: lg.Sugar()}, nil -} - -// NewGRPCLoggerV2FromZapCore creates "grpclog.LoggerV2" from "zap.Core" -// and "zapcore.WriteSyncer". It discards all INFO level logging in gRPC, -// if debug level is not enabled in "*zap.Logger". -func NewGRPCLoggerV2FromZapCore(cr zapcore.Core, syncer zapcore.WriteSyncer) grpclog.LoggerV2 { - // "AddCallerSkip" to annotate caller outside of "logutil" - lg := zap.New(cr, zap.AddCaller(), zap.AddCallerSkip(1), zap.ErrorOutput(syncer)).Named("grpc") - return &zapGRPCLogger{lg: lg, sugar: lg.Sugar()} -} - -type zapGRPCLogger struct { - lg *zap.Logger - sugar *zap.SugaredLogger -} - -func (zl *zapGRPCLogger) Info(args ...interface{}) { - if !zl.lg.Core().Enabled(zapcore.DebugLevel) { - return - } - zl.sugar.Info(args...) -} - -func (zl *zapGRPCLogger) Infoln(args ...interface{}) { - if !zl.lg.Core().Enabled(zapcore.DebugLevel) { - return - } - zl.sugar.Info(args...) -} - -func (zl *zapGRPCLogger) Infof(format string, args ...interface{}) { - if !zl.lg.Core().Enabled(zapcore.DebugLevel) { - return - } - zl.sugar.Infof(format, args...) -} - -func (zl *zapGRPCLogger) Warning(args ...interface{}) { - zl.sugar.Warn(args...) -} - -func (zl *zapGRPCLogger) Warningln(args ...interface{}) { - zl.sugar.Warn(args...) -} - -func (zl *zapGRPCLogger) Warningf(format string, args ...interface{}) { - zl.sugar.Warnf(format, args...) -} - -func (zl *zapGRPCLogger) Error(args ...interface{}) { - zl.sugar.Error(args...) -} - -func (zl *zapGRPCLogger) Errorln(args ...interface{}) { - zl.sugar.Error(args...) -} - -func (zl *zapGRPCLogger) Errorf(format string, args ...interface{}) { - zl.sugar.Errorf(format, args...) -} - -func (zl *zapGRPCLogger) Fatal(args ...interface{}) { - zl.sugar.Fatal(args...) -} - -func (zl *zapGRPCLogger) Fatalln(args ...interface{}) { - zl.sugar.Fatal(args...) -} - -func (zl *zapGRPCLogger) Fatalf(format string, args ...interface{}) { - zl.sugar.Fatalf(format, args...) -} - -func (zl *zapGRPCLogger) V(l int) bool { - // infoLog == 0 - if l <= 0 { // debug level, then we ignore info level in gRPC - return !zl.lg.Core().Enabled(zapcore.DebugLevel) - } - return true -} diff --git a/client/pkg/logutil/zap_grpc_test.go b/client/pkg/logutil/zap_grpc_test.go deleted file mode 100644 index 9e028cac8e64..000000000000 --- a/client/pkg/logutil/zap_grpc_test.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logutil - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func TestNewGRPCLoggerV2(t *testing.T) { - logPath := filepath.Join(os.TempDir(), fmt.Sprintf("test-log-%d", time.Now().UnixNano())) - defer os.RemoveAll(logPath) - - lcfg := zap.Config{ - Level: zap.NewAtomicLevelAt(zap.InfoLevel), - Development: false, - Sampling: &zap.SamplingConfig{ - Initial: 100, - Thereafter: 100, - }, - Encoding: "json", - EncoderConfig: DefaultZapLoggerConfig.EncoderConfig, - OutputPaths: []string{logPath}, - ErrorOutputPaths: []string{logPath}, - } - gl, err := NewGRPCLoggerV2(lcfg) - if err != nil { - t.Fatal(err) - } - - // debug level is not enabled, - // so info level gRPC-side logging is discarded - gl.Info("etcd-logutil-1") - data, err := ioutil.ReadFile(logPath) - if err != nil { - t.Fatal(err) - } - if bytes.Contains(data, []byte("etcd-logutil-1")) { - t.Fatalf("unexpected line %q", string(data)) - } - - gl.Warning("etcd-logutil-2") - data, err = ioutil.ReadFile(logPath) - if err != nil { - t.Fatal(err) - } - if !bytes.Contains(data, []byte("etcd-logutil-2")) { - t.Fatalf("can't find data in log %q", string(data)) - } - if !bytes.Contains(data, []byte("logutil/zap_grpc_test.go:")) { - t.Fatalf("unexpected caller; %q", string(data)) - } -} - -func TestNewGRPCLoggerV2FromZapCore(t *testing.T) { - buf := bytes.NewBuffer(nil) - syncer := zapcore.AddSync(buf) - cr := zapcore.NewCore( - zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), - syncer, - zap.NewAtomicLevelAt(zap.InfoLevel), - ) - - lg := NewGRPCLoggerV2FromZapCore(cr, syncer) - lg.Warning("TestNewGRPCLoggerV2FromZapCore") - txt := buf.String() - if !strings.Contains(txt, "TestNewGRPCLoggerV2FromZapCore") { - t.Fatalf("unexpected log %q", txt) - } -} diff --git a/client/pkg/logutil/zap_test.go b/client/pkg/logutil/zap_test.go new file mode 100644 index 000000000000..fcfd5fc15f31 --- /dev/null +++ b/client/pkg/logutil/zap_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "bytes" + "encoding/json" + "regexp" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type commonLogFields struct { + Level string `json:"level"` + Timestamp string `json:"ts"` + Message string `json:"msg"` +} + +const ( + fractionSecondsPrecision = 6 // MicroSeconds +) + +func TestEncodeTimePrecisionToMicroSeconds(t *testing.T) { + buf := bytes.NewBuffer(nil) + syncer := zapcore.AddSync(buf) + zc := zapcore.NewCore( + zapcore.NewJSONEncoder(DefaultZapLoggerConfig.EncoderConfig), + syncer, + zap.NewAtomicLevelAt(zap.InfoLevel), + ) + + lg := zap.New(zc) + lg.Info("TestZapLog") + fields := commonLogFields{} + require.NoError(t, json.Unmarshal(buf.Bytes(), &fields)) + // example 1: 2024-06-06T23:37:21.948385Z + // example 2 with zone offset: 2024-06-06T16:16:44.176778-0700 + regex := `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)(Z|[+-]\d{4})` + re := regexp.MustCompile(regex) + matches := re.FindStringSubmatch(fields.Timestamp) + require.Equal(t, 3, len(matches)) + require.Equalf(t, fractionSecondsPrecision, len(matches[1]), "unexpected timestamp %s", fields.Timestamp) +} diff --git a/client/pkg/srv/srv_test.go b/client/pkg/srv/srv_test.go index c2ccf9485d91..bc234dfc7986 100644 --- a/client/pkg/srv/srv_test.go +++ b/client/pkg/srv/srv_test.go @@ -228,10 +228,10 @@ func TestSRVDiscover(t *testing.T) { []*net.SRV{ {Target: "a.example.com", Port: 2480}, {Target: "b.example.com", Port: 2480}, - {Target: "c.example.com", Port: 2480}, + {Target: "c.example.com.", Port: 2480}, }, []*net.SRV{}, - []string{"https://a.example.com:2480", "https://b.example.com:2480", "https://c.example.com:2480"}, + []string{"https://a.example.com:2480", "https://b.example.com:2480", "https://c.example.com.:2480"}, false, }, } diff --git a/client/pkg/testutil/leak.go b/client/pkg/testutil/leak.go index f786b5ccd448..32235032a54f 100644 --- a/client/pkg/testutil/leak.go +++ b/client/pkg/testutil/leak.go @@ -16,6 +16,8 @@ import ( "time" ) +// TODO: Replace with https://github.com/uber-go/goleak. + /* CheckLeakedGoroutine verifies tests do not leave any leaky goroutines. It returns true when there are goroutines still @@ -28,10 +30,9 @@ running(leaking) after all tests. } func TestSample(t *testing.T) { - BeforeTest(t) + RegisterLeakDetection(t) ... } - */ func CheckLeakedGoroutine() bool { gs := interestingGoroutines() @@ -94,22 +95,22 @@ func CheckAfterTest(d time.Duration) error { return fmt.Errorf("appears to have leaked %s:\n%s", bad, stacks) } -// BeforeTest is a convenient way to register before-and-after code to a test. -// If you execute BeforeTest, you don't need to explicitly register AfterTest. -func BeforeTest(t TB) { +// RegisterLeakDetection is a convenient way to register before-and-after code to a test. +// If you execute RegisterLeakDetection, you don't need to explicitly register AfterTest. +func RegisterLeakDetection(t TB) { if err := CheckAfterTest(10 * time.Millisecond); err != nil { t.Skip("Found leaked goroutined BEFORE test", err) return } t.Cleanup(func() { - AfterTest(t) + afterTest(t) }) } -// AfterTest is meant to run in a defer that executes after a test completes. +// afterTest is meant to run in a defer that executes after a test completes. // It will detect common goroutine leaks, retrying in case there are goroutines // not synchronously torn down, and fail the test if any goroutines are stuck. -func AfterTest(t TB) { +func afterTest(t TB) { // If test-failed the leaked goroutines list is hidding the real // source of problem. if !t.Failed() { @@ -147,7 +148,8 @@ func interestingGoroutines() (gs []string) { strings.Contains(stack, "runtime.MHeap_Scavenger") || strings.Contains(stack, "rcrypto/internal/boring.(*PublicKeyRSA).finalize") || strings.Contains(stack, "net.(*netFD).Close(") || - strings.Contains(stack, "testing.(*T).Run") { + strings.Contains(stack, "testing.(*T).Run") || + strings.Contains(stack, "crypto/tls.(*certCache).evict") { continue } gs = append(gs, stack) diff --git a/client/pkg/testutil/leak_test.go b/client/pkg/testutil/leak_test.go index fa70782f3d65..71b1c7bf3e6a 100644 --- a/client/pkg/testutil/leak_test.go +++ b/client/pkg/testutil/leak_test.go @@ -35,7 +35,7 @@ func TestMain(m *testing.M) { func TestSample(t *testing.T) { SkipTestIfShortMode(t, "Counting leaked routines is disabled in --short tests") - defer AfterTest(t) + defer afterTest(t) ranSample = true for range make([]struct{}, 100) { go func() { diff --git a/client/pkg/tlsutil/cipher_suites.go b/client/pkg/tlsutil/cipher_suites.go index f278a61f8a04..e1f21755d4b7 100644 --- a/client/pkg/tlsutil/cipher_suites.go +++ b/client/pkg/tlsutil/cipher_suites.go @@ -14,7 +14,10 @@ package tlsutil -import "crypto/tls" +import ( + "crypto/tls" + "fmt" +) // GetCipherSuite returns the corresponding cipher suite, // and boolean value if it is supported. @@ -37,3 +40,17 @@ func GetCipherSuite(s string) (uint16, bool) { } return 0, false } + +// GetCipherSuites returns list of corresponding cipher suite IDs. +func GetCipherSuites(ss []string) ([]uint16, error) { + cs := make([]uint16, len(ss)) + for i, s := range ss { + var ok bool + cs[i], ok = GetCipherSuite(s) + if !ok { + return nil, fmt.Errorf("unexpected TLS cipher suite %q", s) + } + } + + return cs, nil +} diff --git a/client/pkg/tlsutil/versions.go b/client/pkg/tlsutil/versions.go new file mode 100644 index 000000000000..ffcecd8c670f --- /dev/null +++ b/client/pkg/tlsutil/versions.go @@ -0,0 +1,47 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tlsutil + +import ( + "crypto/tls" + "fmt" +) + +type TLSVersion string + +// Constants for TLS versions. +const ( + TLSVersionDefault TLSVersion = "" + TLSVersion12 TLSVersion = "TLS1.2" + TLSVersion13 TLSVersion = "TLS1.3" +) + +// GetTLSVersion returns the corresponding tls.Version or error. +func GetTLSVersion(version string) (uint16, error) { + var v uint16 + + switch version { + case string(TLSVersionDefault): + v = 0 // 0 means let Go decide. + case string(TLSVersion12): + v = tls.VersionTLS12 + case string(TLSVersion13): + v = tls.VersionTLS13 + default: + return 0, fmt.Errorf("unexpected TLS version %q (must be one of: TLS1.2, TLS1.3)", version) + } + + return v, nil +} diff --git a/client/pkg/tlsutil/versions_test.go b/client/pkg/tlsutil/versions_test.go new file mode 100644 index 000000000000..89c7c3f64b7d --- /dev/null +++ b/client/pkg/tlsutil/versions_test.go @@ -0,0 +1,63 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tlsutil + +import ( + "crypto/tls" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetVersion(t *testing.T) { + tests := []struct { + name string + version string + want uint16 + expectError bool + }{ + { + name: "TLS1.2", + version: "TLS1.2", + want: tls.VersionTLS12, + }, + { + name: "TLS1.3", + version: "TLS1.3", + want: tls.VersionTLS13, + }, + { + name: "Empty version", + version: "", + want: 0, + }, + { + name: "Converting invalid version string to TLS version", + version: "not_existing", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTLSVersion(tt.version) + if err != nil { + assert.True(t, tt.expectError, "GetTLSVersion() returned error while expecting success: %v", err) + return + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/client/pkg/transport/keepalive_listener.go b/client/pkg/transport/keepalive_listener.go index 4ff8e7f0010c..d00a93dc19ef 100644 --- a/client/pkg/transport/keepalive_listener.go +++ b/client/pkg/transport/keepalive_listener.go @@ -21,26 +21,29 @@ import ( "time" ) -type keepAliveConn interface { - SetKeepAlive(bool) error - SetKeepAlivePeriod(d time.Duration) error -} - // NewKeepAliveListener returns a listener that listens on the given address. // Be careful when wrap around KeepAliveListener with another Listener if TLSInfo is not nil. // Some pkgs (like go/http) might expect Listener to return TLSConn type to start TLS handshake. // http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html +// +// Note(ahrtr): +// only `net.TCPConn` supports `SetKeepAlive` and `SetKeepAlivePeriod` +// by default, so if you want to wrap multiple layers of net.Listener, +// the `keepaliveListener` should be the one which is closest to the +// original `net.Listener` implementation, namely `TCPListener`. func NewKeepAliveListener(l net.Listener, scheme string, tlscfg *tls.Config) (net.Listener, error) { + kal := &keepaliveListener{ + Listener: l, + } + if scheme == "https" { if tlscfg == nil { return nil, fmt.Errorf("cannot listen on TLS for given listener: KeyFile and CertFile are not presented") } - return newTLSKeepaliveListener(l, tlscfg), nil + return newTLSKeepaliveListener(kal, tlscfg), nil } - return &keepaliveListener{ - Listener: l, - }, nil + return kal, nil } type keepaliveListener struct{ net.Listener } @@ -50,13 +53,38 @@ func (kln *keepaliveListener) Accept() (net.Conn, error) { if err != nil { return nil, err } - kac := c.(keepAliveConn) + + kac, err := createKeepaliveConn(c) + if err != nil { + return nil, fmt.Errorf("create keepalive connection failed, %w", err) + } // detection time: tcp_keepalive_time + tcp_keepalive_probes + tcp_keepalive_intvl // default on linux: 30 + 8 * 30 // default on osx: 30 + 8 * 75 - kac.SetKeepAlive(true) - kac.SetKeepAlivePeriod(30 * time.Second) - return c, nil + if err := kac.SetKeepAlive(true); err != nil { + return nil, fmt.Errorf("SetKeepAlive failed, %w", err) + } + if err := kac.SetKeepAlivePeriod(30 * time.Second); err != nil { + return nil, fmt.Errorf("SetKeepAlivePeriod failed, %w", err) + } + return kac, nil +} + +func createKeepaliveConn(c net.Conn) (*keepAliveConn, error) { + tcpc, ok := c.(*net.TCPConn) + if !ok { + return nil, ErrNotTCP + } + return &keepAliveConn{tcpc}, nil +} + +type keepAliveConn struct { + *net.TCPConn +} + +// SetKeepAlive sets keepalive +func (l *keepAliveConn) SetKeepAlive(doKeepAlive bool) error { + return l.TCPConn.SetKeepAlive(doKeepAlive) } // A tlsKeepaliveListener implements a network listener (net.Listener) for TLS connections. @@ -72,12 +100,6 @@ func (l *tlsKeepaliveListener) Accept() (c net.Conn, err error) { if err != nil { return } - kac := c.(keepAliveConn) - // detection time: tcp_keepalive_time + tcp_keepalive_probes + tcp_keepalive_intvl - // default on linux: 30 + 8 * 30 - // default on osx: 30 + 8 * 75 - kac.SetKeepAlive(true) - kac.SetKeepAlivePeriod(30 * time.Second) c = tls.Server(c, l.config) return c, nil } diff --git a/client/pkg/transport/keepalive_listener_openbsd.go b/client/pkg/transport/keepalive_listener_openbsd.go new file mode 100644 index 000000000000..024c6c23639f --- /dev/null +++ b/client/pkg/transport/keepalive_listener_openbsd.go @@ -0,0 +1,26 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build openbsd + +package transport + +import "time" + +// SetKeepAlivePeriod sets keepalive period +func (l *keepAliveConn) SetKeepAlivePeriod(d time.Duration) error { + // OpenBSD has no user-settable per-socket TCP keepalive options. + // Refer to https://github.com/etcd-io/etcd/issues/15811. + return nil +} diff --git a/client/pkg/transport/keepalive_listener_test.go b/client/pkg/transport/keepalive_listener_test.go index 425f53368b54..041e48953b10 100644 --- a/client/pkg/transport/keepalive_listener_test.go +++ b/client/pkg/transport/keepalive_listener_test.go @@ -40,6 +40,9 @@ func TestNewKeepAliveListener(t *testing.T) { if err != nil { t.Fatalf("unexpected Accept error: %v", err) } + if _, ok := conn.(*keepAliveConn); !ok { + t.Fatalf("Unexpected conn type: %T, wanted *keepAliveConn", conn) + } conn.Close() ln.Close() diff --git a/client/pkg/transport/keepalive_listener_unix.go b/client/pkg/transport/keepalive_listener_unix.go new file mode 100644 index 000000000000..08061f7267b2 --- /dev/null +++ b/client/pkg/transport/keepalive_listener_unix.go @@ -0,0 +1,24 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !openbsd + +package transport + +import "time" + +// SetKeepAlivePeriod sets keepalive period +func (l *keepAliveConn) SetKeepAlivePeriod(d time.Duration) error { + return l.TCPConn.SetKeepAlivePeriod(d) +} diff --git a/client/pkg/transport/limit_listen.go b/client/pkg/transport/limit_listen.go index 930c542066f8..404722ba76ed 100644 --- a/client/pkg/transport/limit_listen.go +++ b/client/pkg/transport/limit_listen.go @@ -63,6 +63,9 @@ func (l *limitListenerConn) Close() error { return err } +// SetKeepAlive sets keepalive +// +// Deprecated: use (*keepAliveConn) SetKeepAlive instead. func (l *limitListenerConn) SetKeepAlive(doKeepAlive bool) error { tcpc, ok := l.Conn.(*net.TCPConn) if !ok { @@ -71,6 +74,9 @@ func (l *limitListenerConn) SetKeepAlive(doKeepAlive bool) error { return tcpc.SetKeepAlive(doKeepAlive) } +// SetKeepAlivePeriod sets keepalive period +// +// Deprecated: use (*keepAliveConn) SetKeepAlivePeriod instead. func (l *limitListenerConn) SetKeepAlivePeriod(d time.Duration) error { tcpc, ok := l.Conn.(*net.TCPConn) if !ok { diff --git a/client/pkg/transport/listener.go b/client/pkg/transport/listener.go index 992c773eaacc..a7d37688d92f 100644 --- a/client/pkg/transport/listener.go +++ b/client/pkg/transport/listener.go @@ -68,7 +68,7 @@ func newListener(addr, scheme string, opts ...ListenerOption) (net.Listener, err fallthrough case lnOpts.IsTimeout(), lnOpts.IsSocketOpts(): // timeout listener with socket options. - ln, err := lnOpts.ListenConfig.Listen(context.TODO(), "tcp", addr) + ln, err := newKeepAliveListener(&lnOpts.ListenConfig, addr) if err != nil { return nil, err } @@ -78,7 +78,7 @@ func newListener(addr, scheme string, opts ...ListenerOption) (net.Listener, err writeTimeout: lnOpts.writeTimeout, } case lnOpts.IsTimeout(): - ln, err := net.Listen("tcp", addr) + ln, err := newKeepAliveListener(nil, addr) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func newListener(addr, scheme string, opts ...ListenerOption) (net.Listener, err writeTimeout: lnOpts.writeTimeout, } default: - ln, err := net.Listen("tcp", addr) + ln, err := newKeepAliveListener(nil, addr) if err != nil { return nil, err } @@ -102,6 +102,19 @@ func newListener(addr, scheme string, opts ...ListenerOption) (net.Listener, err return wrapTLS(scheme, lnOpts.tlsInfo, lnOpts.Listener) } +func newKeepAliveListener(cfg *net.ListenConfig, addr string) (ln net.Listener, err error) { + if cfg != nil { + ln, err = cfg.Listen(context.TODO(), "tcp", addr) + } else { + ln, err = net.Listen("tcp", addr) + } + if err != nil { + return + } + + return NewKeepAliveListener(ln, "tcp", nil) +} + func wrapTLS(scheme string, tlsinfo *TLSInfo, l net.Listener) (net.Listener, error) { if scheme != "https" && scheme != "unixs" { return l, nil @@ -152,6 +165,14 @@ type TLSInfo struct { // Note that cipher suites are prioritized in the given order. CipherSuites []uint16 + // MinVersion is the minimum TLS version that is acceptable. + // If not set, the minimum version is TLS 1.2. + MinVersion uint16 + + // MaxVersion is the maximum TLS version that is acceptable. + // If not set, the default used by Go is selected (see tls.Config.MaxVersion). + MaxVersion uint16 + selfCert bool // parseFunc exists to simplify testing. Typically, parseFunc @@ -159,12 +180,23 @@ type TLSInfo struct { parseFunc func([]byte, []byte) (tls.Certificate, error) // AllowedCN is a CN which must be provided by a client. + // + // Deprecated: use AllowedCNs instead. AllowedCN string // AllowedHostname is an IP address or hostname that must match the TLS // certificate provided by a client. + // + // Deprecated: use AllowedHostnames instead. AllowedHostname string + // AllowedCNs is a list of acceptable CNs which must be provided by a client. + AllowedCNs []string + + // AllowedHostnames is a list of acceptable IP addresses or hostnames that must match the + // TLS certificate provided by a client. + AllowedHostnames []string + // Logger logs TLS errors. // If nil, all logs are discarded. Logger *zap.Logger @@ -192,7 +224,7 @@ func SelfCert(lg *zap.Logger, dirpath string, hosts []string, selfSignedCertVali ) return } - err = fileutil.TouchDirAll(dirpath) + err = fileutil.TouchDirAll(lg, dirpath) if err != nil { if info.Logger != nil { info.Logger.Warn( @@ -326,8 +358,8 @@ func SelfCert(lg *zap.Logger, dirpath string, hosts []string, selfSignedCertVali // Previously, // 1. Server has non-empty (*tls.Config).Certificates on client hello // 2. Server calls (*tls.Config).GetCertificate iff: -// - Server's (*tls.Config).Certificates is not empty, or -// - Client supplies SNI; non-empty (*tls.ClientHelloInfo).ServerName +// - Server's (*tls.Config).Certificates is not empty, or +// - Client supplies SNI; non-empty (*tls.ClientHelloInfo).ServerName // // When (*tls.Config).Certificates is always populated on initial handshake, // client is expected to provide a valid matching SNI to pass the TLS @@ -365,8 +397,17 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { } } + var minVersion uint16 + if info.MinVersion != 0 { + minVersion = info.MinVersion + } else { + // Default minimum version is TLS 1.2, previous versions are insecure and deprecated. + minVersion = tls.VersionTLS12 + } + cfg := &tls.Config{ - MinVersion: tls.VersionTLS12, + MinVersion: minVersion, + MaxVersion: info.MaxVersion, ServerName: info.ServerName, } @@ -377,19 +418,52 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { // Client certificates may be verified by either an exact match on the CN, // or a more general check of the CN and SANs. var verifyCertificate func(*x509.Certificate) bool + + if info.AllowedCN != "" && len(info.AllowedCNs) > 0 { + return nil, fmt.Errorf("AllowedCN and AllowedCNs are mutually exclusive (cn=%q, cns=%q)", info.AllowedCN, info.AllowedCNs) + } + if info.AllowedHostname != "" && len(info.AllowedHostnames) > 0 { + return nil, fmt.Errorf("AllowedHostname and AllowedHostnames are mutually exclusive (hostname=%q, hostnames=%q)", info.AllowedHostname, info.AllowedHostnames) + } + if info.AllowedCN != "" && info.AllowedHostname != "" { + return nil, fmt.Errorf("AllowedCN and AllowedHostname are mutually exclusive (cn=%q, hostname=%q)", info.AllowedCN, info.AllowedHostname) + } + if len(info.AllowedCNs) > 0 && len(info.AllowedHostnames) > 0 { + return nil, fmt.Errorf("AllowedCNs and AllowedHostnames are mutually exclusive (cns=%q, hostnames=%q)", info.AllowedCNs, info.AllowedHostnames) + } + if info.AllowedCN != "" { - if info.AllowedHostname != "" { - return nil, fmt.Errorf("AllowedCN and AllowedHostname are mutually exclusive (cn=%q, hostname=%q)", info.AllowedCN, info.AllowedHostname) - } + info.Logger.Warn("AllowedCN is deprecated, use AllowedCNs instead") verifyCertificate = func(cert *x509.Certificate) bool { return info.AllowedCN == cert.Subject.CommonName } } if info.AllowedHostname != "" { + info.Logger.Warn("AllowedHostname is deprecated, use AllowedHostnames instead") verifyCertificate = func(cert *x509.Certificate) bool { return cert.VerifyHostname(info.AllowedHostname) == nil } } + if len(info.AllowedCNs) > 0 { + verifyCertificate = func(cert *x509.Certificate) bool { + for _, allowedCN := range info.AllowedCNs { + if allowedCN == cert.Subject.CommonName { + return true + } + } + return false + } + } + if len(info.AllowedHostnames) > 0 { + verifyCertificate = func(cert *x509.Certificate) bool { + for _, allowedHostname := range info.AllowedHostnames { + if cert.VerifyHostname(allowedHostname) == nil { + return true + } + } + return false + } + } if verifyCertificate != nil { cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { for _, chains := range verifiedChains { @@ -497,11 +571,6 @@ func (info TLSInfo) ServerConfig() (*tls.Config, error) { // "h2" NextProtos is necessary for enabling HTTP2 for go's HTTP server cfg.NextProtos = []string{"h2"} - // go1.13 enables TLS 1.3 by default - // and in TLS 1.3, cipher suites are not configurable - // setting Max TLS version to TLS 1.2 for go 1.13 - cfg.MaxVersion = tls.VersionTLS12 - return cfg, nil } @@ -556,11 +625,6 @@ func (info TLSInfo) ClientConfig() (*tls.Config, error) { } } - // go1.13 enables TLS 1.3 by default - // and in TLS 1.3, cipher suites are not configurable - // setting Max TLS version to TLS 1.2 for go 1.13 - cfg.MaxVersion = tls.VersionTLS12 - return cfg, nil } diff --git a/client/pkg/transport/listener_test.go b/client/pkg/transport/listener_test.go index 00657648ece2..96d0563da0ee 100644 --- a/client/pkg/transport/listener_test.go +++ b/client/pkg/transport/listener_test.go @@ -213,6 +213,15 @@ func TestNewListenerWithSocketOpts(t *testing.T) { if !test.expectedErr && err != nil { t.Fatalf("unexpected error: %v", err) } + + if test.scheme == "http" { + lnOpts := newListenOpts(test.opts...) + if !lnOpts.IsSocketOpts() && !lnOpts.IsTimeout() { + if _, ok := ln.(*keepaliveListener); !ok { + t.Fatalf("ln: unexpected listener type: %T, wanted *keepaliveListener", ln) + } + } + } }) } } diff --git a/client/pkg/transport/sockopt.go b/client/pkg/transport/sockopt.go index 38548ddd7131..fe028c613232 100644 --- a/client/pkg/transport/sockopt.go +++ b/client/pkg/transport/sockopt.go @@ -21,12 +21,12 @@ type SocketOpts struct { // in which case lock on data file could result in unexpected // condition. User should take caution to protect against lock race. // [1] https://man7.org/linux/man-pages/man7/socket.7.html - ReusePort bool + ReusePort bool `json:"reuse-port"` // ReuseAddress enables a socket option SO_REUSEADDR which allows // binding to an address in `TIME_WAIT` state. Useful to improve MTTR // in cases where etcd slow to restart due to excessive `TIME_WAIT`. // [1] https://man7.org/linux/man-pages/man7/socket.7.html - ReuseAddress bool + ReuseAddress bool `json:"reuse-address"` } func getControls(sopts *SocketOpts) Controls { diff --git a/client/pkg/transport/sockopt_solaris.go b/client/pkg/transport/sockopt_solaris.go new file mode 100644 index 000000000000..495c736365e3 --- /dev/null +++ b/client/pkg/transport/sockopt_solaris.go @@ -0,0 +1,35 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build solaris +// +build solaris + +package transport + +import ( + "fmt" + "syscall" + + "golang.org/x/sys/unix" +) + +func setReusePort(network, address string, c syscall.RawConn) error { + return fmt.Errorf("port reuse is not supported on Solaris") +} + +func setReuseAddress(network, address string, conn syscall.RawConn) error { + return conn.Control(func(fd uintptr) { + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1) + }) +} diff --git a/client/pkg/transport/sockopt_unix.go b/client/pkg/transport/sockopt_unix.go index 432b52e0fcee..e2cc6f482863 100644 --- a/client/pkg/transport/sockopt_unix.go +++ b/client/pkg/transport/sockopt_unix.go @@ -1,5 +1,19 @@ -//go:build !windows -// +build !windows +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows && !solaris +// +build !windows,!solaris package transport diff --git a/client/pkg/transport/tls.go b/client/pkg/transport/tls.go index 62fe0d385195..8c3a35b140bb 100644 --- a/client/pkg/transport/tls.go +++ b/client/pkg/transport/tls.go @@ -15,6 +15,7 @@ package transport import ( + "context" "fmt" "strings" "time" @@ -27,6 +28,8 @@ func ValidateSecureEndpoints(tlsInfo TLSInfo, eps []string) ([]string, error) { if err != nil { return nil, err } + defer t.CloseIdleConnections() + var errs []string var endpoints []string for _, ep := range eps { @@ -34,7 +37,7 @@ func ValidateSecureEndpoints(tlsInfo TLSInfo, eps []string) ([]string, error) { errs = append(errs, fmt.Sprintf("%q is insecure", ep)) continue } - conn, cerr := t.Dial("tcp", ep[len("https://"):]) + conn, cerr := t.DialContext(context.Background(), "tcp", ep[len("https://"):]) if cerr != nil { errs = append(errs, fmt.Sprintf("%q failed to dial (%v)", ep, cerr)) continue diff --git a/client/pkg/transport/tls_test.go b/client/pkg/transport/tls_test.go new file mode 100644 index 000000000000..8fb80b126761 --- /dev/null +++ b/client/pkg/transport/tls_test.go @@ -0,0 +1,90 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestValidateSecureEndpoints(t *testing.T) { + tlsInfo, certCleanup, err := createSelfCert() + if err != nil { + t.Fatalf("unable to create cert: %v", err) + } + defer certCleanup() + + remoteAddr := func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(r.RemoteAddr)) + } + srv := httptest.NewServer(http.HandlerFunc(remoteAddr)) + defer srv.Close() + + tests := map[string]struct { + endPoints []string + expectedEndpoints []string + expectedErr bool + }{ + "invalidEndPoints": { + endPoints: []string{ + "invalid endpoint", + }, + expectedEndpoints: nil, + expectedErr: true, + }, + "insecureEndpoints": { + endPoints: []string{ + "http://127.0.0.1:8000", + "http://" + srv.Listener.Addr().String(), + }, + expectedEndpoints: nil, + expectedErr: true, + }, + "secureEndPoints": { + endPoints: []string{ + "https://" + srv.Listener.Addr().String(), + }, + expectedEndpoints: []string{ + "https://" + srv.Listener.Addr().String(), + }, + expectedErr: false, + }, + "mixEndPoints": { + endPoints: []string{ + "https://" + srv.Listener.Addr().String(), + "http://" + srv.Listener.Addr().String(), + "invalid end points", + }, + expectedEndpoints: []string{ + "https://" + srv.Listener.Addr().String(), + }, + expectedErr: true, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + secureEps, err := ValidateSecureEndpoints(*tlsInfo, test.endPoints) + if test.expectedErr != (err != nil) { + t.Errorf("Unexpected error, got: %v, want: %v", err, test.expectedErr) + } + + if !reflect.DeepEqual(test.expectedEndpoints, secureEps) { + t.Errorf("expected endpoints %v, got %v", test.expectedEndpoints, secureEps) + } + }) + } +} diff --git a/client/pkg/types/urls.go b/client/pkg/types/urls.go index 9e5d03ff6457..49a38967e64d 100644 --- a/client/pkg/types/urls.go +++ b/client/pkg/types/urls.go @@ -36,20 +36,25 @@ func NewURLs(strs []string) (URLs, error) { if err != nil { return nil, err } - if u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "unix" && u.Scheme != "unixs" { + + switch u.Scheme { + case "http", "https": + if _, _, err := net.SplitHostPort(u.Host); err != nil { + return nil, fmt.Errorf(`URL address does not have the form "host:port": %s`, in) + } + + if u.Path != "" { + return nil, fmt.Errorf("URL must not contain a path: %s", in) + } + case "unix", "unixs": + break + default: return nil, fmt.Errorf("URL scheme must be http, https, unix, or unixs: %s", in) } - if _, _, err := net.SplitHostPort(u.Host); err != nil { - return nil, fmt.Errorf(`URL address does not have the form "host:port": %s`, in) - } - if u.Path != "" { - return nil, fmt.Errorf("URL must not contain a path: %s", in) - } all[i] = *u } us := URLs(all) us.Sort() - return us, nil } diff --git a/client/v2/doc.go b/client/v2/doc.go index 5250758b017c..68284c20a89c 100644 --- a/client/v2/doc.go +++ b/client/v2/doc.go @@ -68,6 +68,5 @@ Use a custom context to set timeouts on your operations: // handle error } } - */ package client diff --git a/client/v2/go.mod b/client/v2/go.mod index d6880b3f85b4..34b21493251b 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -1,12 +1,19 @@ module go.etcd.io/etcd/client/v2 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 require ( - github.com/json-iterator/go v1.1.10 + github.com/json-iterator/go v1.1.11 github.com/modern-go/reflect2 v1.0.1 - go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 + go.etcd.io/etcd/api/v3 v3.5.21 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 +) + +require ( + github.com/coreos/go-semver v0.3.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect ) replace ( diff --git a/client/v2/go.sum b/client/v2/go.sum index ef8988b078e2..e6e8e38851c9 100644 --- a/client/v2/go.sum +++ b/client/v2/go.sum @@ -1,145 +1,24 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/v3/README.md b/client/v3/README.md index 1e037d7eb6b5..16c0fe888cf7 100644 --- a/client/v3/README.md +++ b/client/v3/README.md @@ -11,13 +11,6 @@ go get go.etcd.io/etcd/client/v3 ``` -Warning: As etcd 3.5.0 was not yet released, the command above does not work. -After first pre-release of 3.5.0 [#12498](https://github.com/etcd-io/etcd/issues/12498), -etcd can be referenced using: -``` -go get go.etcd.io/etcd/client/v3@v3.5.0-pre -``` - ## Get started Create client using `clientv3.New`: diff --git a/client/v3/auth.go b/client/v3/auth.go index a6f75d321592..110918a4c7c4 100644 --- a/client/v3/auth.go +++ b/client/v3/auth.go @@ -134,67 +134,67 @@ func NewAuthFromAuthClient(remote pb.AuthClient, c *Client) Auth { func (auth *authClient) Authenticate(ctx context.Context, name string, password string) (*AuthenticateResponse, error) { resp, err := auth.remote.Authenticate(ctx, &pb.AuthenticateRequest{Name: name, Password: password}, auth.callOpts...) - return (*AuthenticateResponse)(resp), toErr(ctx, err) + return (*AuthenticateResponse)(resp), ContextError(ctx, err) } func (auth *authClient) AuthEnable(ctx context.Context) (*AuthEnableResponse, error) { resp, err := auth.remote.AuthEnable(ctx, &pb.AuthEnableRequest{}, auth.callOpts...) - return (*AuthEnableResponse)(resp), toErr(ctx, err) + return (*AuthEnableResponse)(resp), ContextError(ctx, err) } func (auth *authClient) AuthDisable(ctx context.Context) (*AuthDisableResponse, error) { resp, err := auth.remote.AuthDisable(ctx, &pb.AuthDisableRequest{}, auth.callOpts...) - return (*AuthDisableResponse)(resp), toErr(ctx, err) + return (*AuthDisableResponse)(resp), ContextError(ctx, err) } func (auth *authClient) AuthStatus(ctx context.Context) (*AuthStatusResponse, error) { resp, err := auth.remote.AuthStatus(ctx, &pb.AuthStatusRequest{}, auth.callOpts...) - return (*AuthStatusResponse)(resp), toErr(ctx, err) + return (*AuthStatusResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) { resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password, Options: &authpb.UserAddOptions{NoPassword: false}}, auth.callOpts...) - return (*AuthUserAddResponse)(resp), toErr(ctx, err) + return (*AuthUserAddResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserAddWithOptions(ctx context.Context, name string, password string, options *UserAddOptions) (*AuthUserAddResponse, error) { resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password, Options: (*authpb.UserAddOptions)(options)}, auth.callOpts...) - return (*AuthUserAddResponse)(resp), toErr(ctx, err) + return (*AuthUserAddResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error) { resp, err := auth.remote.UserDelete(ctx, &pb.AuthUserDeleteRequest{Name: name}, auth.callOpts...) - return (*AuthUserDeleteResponse)(resp), toErr(ctx, err) + return (*AuthUserDeleteResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error) { resp, err := auth.remote.UserChangePassword(ctx, &pb.AuthUserChangePasswordRequest{Name: name, Password: password}, auth.callOpts...) - return (*AuthUserChangePasswordResponse)(resp), toErr(ctx, err) + return (*AuthUserChangePasswordResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error) { resp, err := auth.remote.UserGrantRole(ctx, &pb.AuthUserGrantRoleRequest{User: user, Role: role}, auth.callOpts...) - return (*AuthUserGrantRoleResponse)(resp), toErr(ctx, err) + return (*AuthUserGrantRoleResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error) { resp, err := auth.remote.UserGet(ctx, &pb.AuthUserGetRequest{Name: name}, auth.callOpts...) - return (*AuthUserGetResponse)(resp), toErr(ctx, err) + return (*AuthUserGetResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserList(ctx context.Context) (*AuthUserListResponse, error) { resp, err := auth.remote.UserList(ctx, &pb.AuthUserListRequest{}, auth.callOpts...) - return (*AuthUserListResponse)(resp), toErr(ctx, err) + return (*AuthUserListResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error) { resp, err := auth.remote.UserRevokeRole(ctx, &pb.AuthUserRevokeRoleRequest{Name: name, Role: role}, auth.callOpts...) - return (*AuthUserRevokeRoleResponse)(resp), toErr(ctx, err) + return (*AuthUserRevokeRoleResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error) { resp, err := auth.remote.RoleAdd(ctx, &pb.AuthRoleAddRequest{Name: name}, auth.callOpts...) - return (*AuthRoleAddResponse)(resp), toErr(ctx, err) + return (*AuthRoleAddResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error) { @@ -204,27 +204,27 @@ func (auth *authClient) RoleGrantPermission(ctx context.Context, name string, ke PermType: authpb.Permission_Type(permType), } resp, err := auth.remote.RoleGrantPermission(ctx, &pb.AuthRoleGrantPermissionRequest{Name: name, Perm: perm}, auth.callOpts...) - return (*AuthRoleGrantPermissionResponse)(resp), toErr(ctx, err) + return (*AuthRoleGrantPermissionResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error) { resp, err := auth.remote.RoleGet(ctx, &pb.AuthRoleGetRequest{Role: role}, auth.callOpts...) - return (*AuthRoleGetResponse)(resp), toErr(ctx, err) + return (*AuthRoleGetResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleList(ctx context.Context) (*AuthRoleListResponse, error) { resp, err := auth.remote.RoleList(ctx, &pb.AuthRoleListRequest{}, auth.callOpts...) - return (*AuthRoleListResponse)(resp), toErr(ctx, err) + return (*AuthRoleListResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error) { resp, err := auth.remote.RoleRevokePermission(ctx, &pb.AuthRoleRevokePermissionRequest{Role: role, Key: []byte(key), RangeEnd: []byte(rangeEnd)}, auth.callOpts...) - return (*AuthRoleRevokePermissionResponse)(resp), toErr(ctx, err) + return (*AuthRoleRevokePermissionResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error) { resp, err := auth.remote.RoleDelete(ctx, &pb.AuthRoleDeleteRequest{Role: role}, auth.callOpts...) - return (*AuthRoleDeleteResponse)(resp), toErr(ctx, err) + return (*AuthRoleDeleteResponse)(resp), ContextError(ctx, err) } func StrToPermissionType(s string) (PermissionType, error) { diff --git a/client/v3/client.go b/client/v3/client.go index 75e3c97e22a5..f7aa65a0a729 100644 --- a/client/v3/client.go +++ b/client/v3/client.go @@ -24,6 +24,7 @@ import ( "time" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/v3/credentials" "go.etcd.io/etcd/client/v3/internal/endpoint" "go.etcd.io/etcd/client/v3/internal/resolver" @@ -85,7 +86,7 @@ func New(cfg Config) (*Client, error) { // service interface implementations and do not need connection management. func NewCtxClient(ctx context.Context, opts ...Option) *Client { cctx, cancel := context.WithCancel(ctx) - c := &Client{ctx: cctx, cancel: cancel, lgMu: new(sync.RWMutex)} + c := &Client{ctx: cctx, cancel: cancel, lgMu: new(sync.RWMutex), mu: new(sync.RWMutex)} for _, opt := range opts { opt(c) } @@ -147,7 +148,7 @@ func (c *Client) Close() error { c.Lease.Close() } if c.conn != nil { - return toErr(c.ctx, c.conn.Close()) + return ContextError(c.ctx, c.conn.Close()) } return c.ctx.Err() } @@ -184,7 +185,9 @@ func (c *Client) Sync(ctx context.Context) error { } var eps []string for _, m := range mresp.Members { - eps = append(eps, m.ClientURLs...) + if len(m.Name) != 0 && !m.IsLearner { + eps = append(eps, m.ClientURLs...) + } } c.SetEndpoints(eps...) return nil @@ -228,15 +231,30 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts opts = append(opts, grpc.WithInsecure()) } + unaryMaxRetries := defaultUnaryMaxRetries + if c.cfg.MaxUnaryRetries > 0 { + unaryMaxRetries = c.cfg.MaxUnaryRetries + } + + backoffWaitBetween := defaultBackoffWaitBetween + if c.cfg.BackoffWaitBetween > 0 { + backoffWaitBetween = c.cfg.BackoffWaitBetween + } + + backoffJitterFraction := defaultBackoffJitterFraction + if c.cfg.BackoffJitterFraction > 0 { + backoffJitterFraction = c.cfg.BackoffJitterFraction + } + // Interceptor retry and backoff. // TODO: Replace all of clientv3/retry.go with RetryPolicy: // https://github.com/grpc/grpc-proto/blob/cdd9ed5c3d3f87aef62f373b93361cf7bddc620d/grpc/service_config/service_config.proto#L130 - rrBackoff := withBackoff(c.roundRobinQuorumBackoff(defaultBackoffWaitBetween, defaultBackoffJitterFraction)) + rrBackoff := withBackoff(c.roundRobinQuorumBackoff(backoffWaitBetween, backoffJitterFraction)) opts = append(opts, // Disable stream retry by default since go-grpc-middleware/retry does not support client streams. // Streams that are safe to retry are enabled individually. grpc.WithStreamInterceptor(c.streamClientInterceptor(withMax(0), rrBackoff)), - grpc.WithUnaryInterceptor(c.unaryClientInterceptor(withMax(defaultUnaryMaxRetries), rrBackoff)), + grpc.WithUnaryInterceptor(c.unaryClientInterceptor(withMax(unaryMaxRetries), rrBackoff)), ) return opts, nil @@ -261,6 +279,7 @@ func (c *Client) getToken(ctx context.Context) error { resp, err := c.Auth.Authenticate(ctx, c.Username, c.Password) if err != nil { if err == rpctypes.ErrAuthNotEnabled { + c.authTokenBundle.UpdateAuthToken("") return nil } return err @@ -283,8 +302,7 @@ func (c *Client) dial(creds grpccredentials.TransportCredentials, dopts ...grpc. if err != nil { return nil, fmt.Errorf("failed to configure dialer: %v", err) } - if c.Username != "" && c.Password != "" { - c.authTokenBundle = credentials.NewBundle(credentials.Config{}) + if c.authTokenBundle != nil { opts = append(opts, grpc.WithPerRPCCredentials(c.authTokenBundle.PerRPCCredentials())) } @@ -296,9 +314,7 @@ func (c *Client) dial(creds grpccredentials.TransportCredentials, dopts ...grpc. dctx, cancel = context.WithTimeout(c.ctx, c.cfg.DialTimeout) defer cancel() // TODO: Is this right for cases where grpc.WithBlock() is not set on the dial options? } - - initialEndpoints := strings.Join(c.cfg.Endpoints, ";") - target := fmt.Sprintf("%s://%p/#initially=[%s]", resolver.Schema, c, initialEndpoints) + target := fmt.Sprintf("%s://%p/%s", resolver.Schema, c, authority(c.Endpoints()[0])) conn, err := grpc.DialContext(dctx, target, opts...) if err != nil { return nil, err @@ -306,6 +322,20 @@ func (c *Client) dial(creds grpccredentials.TransportCredentials, dopts ...grpc. return conn, nil } +func authority(endpoint string) string { + spl := strings.SplitN(endpoint, "://", 2) + if len(spl) < 2 { + if strings.HasPrefix(endpoint, "unix:") { + return endpoint[len("unix:"):] + } + if strings.HasPrefix(endpoint, "unixs:") { + return endpoint[len("unixs:"):] + } + return endpoint + } + return spl[1] +} + func (c *Client) credentialsForEndpoint(ep string) grpccredentials.TransportCredentials { r := endpoint.RequiresCredentials(ep) switch r { @@ -356,7 +386,10 @@ func newClient(cfg *Config) (*Client, error) { } else if cfg.LogConfig != nil { client.lg, err = cfg.LogConfig.Build() } else { - client.lg, err = createDefaultZapLogger() + client.lg, err = logutil.CreateDefaultZapLogger(etcdClientDebugLevel()) + if client.lg != nil { + client.lg = client.lg.Named("etcd-client") + } } if err != nil { return nil, err @@ -365,6 +398,7 @@ func newClient(cfg *Config) (*Client, error) { if cfg.Username != "" && cfg.Password != "" { client.Username = cfg.Username client.Password = cfg.Password + client.authTokenBundle = credentials.NewBundle(credentials.Config{}) } if cfg.MaxCallSendMsgSize > 0 || cfg.MaxCallRecvMsgSize > 0 { if cfg.MaxCallRecvMsgSize > 0 && cfg.MaxCallSendMsgSize > cfg.MaxCallRecvMsgSize { @@ -483,7 +517,7 @@ func (c *Client) checkVersion() (err error) { return } } - if maj < 3 || (maj == 3 && min < 2) { + if maj < 3 || (maj == 3 && min < 4) { rerr = ErrOldCluster } errc <- rerr @@ -491,7 +525,7 @@ func (c *Client) checkVersion() (err error) { } // wait for success for range eps { - if err = <-errc; err == nil { + if err = <-errc; err != nil { break } } @@ -539,7 +573,9 @@ func isUnavailableErr(ctx context.Context, err error) bool { return false } -func toErr(ctx context.Context, err error) error { +// ContextError converts the error into an EtcdError if the error message matches one of +// the defined messages; otherwise, it tries to retrieve the context error. +func ContextError(ctx context.Context, err error) error { if err == nil { return nil } diff --git a/client/v3/client_test.go b/client/v3/client_test.go index b2ff7ef177c2..b9d12c058985 100644 --- a/client/v3/client_test.go +++ b/client/v3/client_test.go @@ -17,13 +17,16 @@ package clientv3 import ( "context" "fmt" - "go.uber.org/zap" + "io" "net" + "sync" "testing" "time" + "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/testutil" + "go.uber.org/zap" "go.uber.org/zap/zaptest" "google.golang.org/grpc" @@ -35,7 +38,7 @@ func NewClient(t *testing.T, cfg Config) (*Client, error) { } func TestDialCancel(t *testing.T) { - testutil.BeforeTest(t) + testutil.RegisterLeakDetection(t) // accept first connection so client is created with dial timeout ln, err := net.Listen("unix", "dialcancel:12345") @@ -87,7 +90,7 @@ func TestDialCancel(t *testing.T) { } func TestDialTimeout(t *testing.T) { - testutil.BeforeTest(t) + testutil.RegisterLeakDetection(t) wantError := context.DeadlineExceeded @@ -146,6 +149,57 @@ func TestDialNoTimeout(t *testing.T) { c.Close() } +func TestMaxUnaryRetries(t *testing.T) { + maxUnaryRetries := uint(10) + cfg := Config{ + Endpoints: []string{"127.0.0.1:12345"}, + MaxUnaryRetries: maxUnaryRetries, + } + c, err := NewClient(t, cfg) + if c == nil || err != nil { + t.Fatalf("new client with MaxUnaryRetries should succeed, got %v", err) + } + defer c.Close() + + if c.cfg.MaxUnaryRetries != maxUnaryRetries { + t.Fatalf("client MaxUnaryRetries should be %d, got %d", maxUnaryRetries, c.cfg.MaxUnaryRetries) + } +} + +func TestBackoff(t *testing.T) { + backoffWaitBetween := 100 * time.Millisecond + cfg := Config{ + Endpoints: []string{"127.0.0.1:12345"}, + BackoffWaitBetween: backoffWaitBetween, + } + c, err := NewClient(t, cfg) + if c == nil || err != nil { + t.Fatalf("new client with BackoffWaitBetween should succeed, got %v", err) + } + defer c.Close() + + if c.cfg.BackoffWaitBetween != backoffWaitBetween { + t.Fatalf("client BackoffWaitBetween should be %v, got %v", backoffWaitBetween, c.cfg.BackoffWaitBetween) + } +} + +func TestBackoffJitterFraction(t *testing.T) { + backoffJitterFraction := float64(0.9) + cfg := Config{ + Endpoints: []string{"127.0.0.1:12345"}, + BackoffJitterFraction: backoffJitterFraction, + } + c, err := NewClient(t, cfg) + if c == nil || err != nil { + t.Fatalf("new client with BackoffJitterFraction should succeed, got %v", err) + } + defer c.Close() + + if c.cfg.BackoffJitterFraction != backoffJitterFraction { + t.Fatalf("client BackoffJitterFraction should be %v, got %v", backoffJitterFraction, c.cfg.BackoffJitterFraction) + } +} + func TestIsHaltErr(t *testing.T) { if !isHaltErr(context.TODO(), fmt.Errorf("etcdserver: some etcdserver error")) { t.Errorf(`error prefixed with "etcdserver: " should be Halted by default`) @@ -198,3 +252,194 @@ func TestZapWithLogger(t *testing.T) { t.Errorf("WithZapLogger should modify *zap.Logger") } } + +func TestAuthTokenBundleNoOverwrite(t *testing.T) { + // Create a mock AuthServer to handle Authenticate RPCs. + lis, err := net.Listen("unix", "etcd-auth-test:0") + if err != nil { + t.Fatal(err) + } + defer lis.Close() + addr := "unix:" + lis.Addr().String() + srv := grpc.NewServer() + etcdserverpb.RegisterAuthServer(srv, mockAuthServer{}) + go srv.Serve(lis) + defer srv.Stop() + + // Create a client, which should call Authenticate on the mock server to + // exchange username/password for an auth token. + c, err := NewClient(t, Config{ + DialTimeout: 5 * time.Second, + Endpoints: []string{addr}, + Username: "foo", + Password: "bar", + }) + if err != nil { + t.Fatal(err) + } + defer c.Close() + oldTokenBundle := c.authTokenBundle + + // Call the public Dial again, which should preserve the original + // authTokenBundle. + gc, err := c.Dial(addr) + if err != nil { + t.Fatal(err) + } + defer gc.Close() + newTokenBundle := c.authTokenBundle + + if oldTokenBundle != newTokenBundle { + t.Error("Client.authTokenBundle has been overwritten during Client.Dial") + } +} + +type mockAuthServer struct { + *etcdserverpb.UnimplementedAuthServer +} + +func (mockAuthServer) Authenticate(context.Context, *etcdserverpb.AuthenticateRequest) (*etcdserverpb.AuthenticateResponse, error) { + return &etcdserverpb.AuthenticateResponse{Token: "mock-token"}, nil +} + +func TestSyncFiltersMembers(t *testing.T) { + c, _ := NewClient(t, Config{Endpoints: []string{"http://254.0.0.1:12345"}}) + defer c.Close() + c.Cluster = &mockCluster{ + []*etcdserverpb.Member{ + {ID: 0, Name: "", ClientURLs: []string{"http://254.0.0.1:12345"}, IsLearner: false}, + {ID: 1, Name: "isStarted", ClientURLs: []string{"http://254.0.0.2:12345"}, IsLearner: true}, + {ID: 2, Name: "isStartedAndNotLearner", ClientURLs: []string{"http://254.0.0.3:12345"}, IsLearner: false}, + }, + } + c.Sync(context.Background()) + + endpoints := c.Endpoints() + if len(endpoints) != 1 || endpoints[0] != "http://254.0.0.3:12345" { + t.Error("Client.Sync uses learner and/or non-started member client URLs") + } +} + +type mockCluster struct { + members []*etcdserverpb.Member +} + +func (mc *mockCluster) MemberList(ctx context.Context) (*MemberListResponse, error) { + return &MemberListResponse{Members: mc.members}, nil +} + +func (mc *mockCluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { + return nil, nil +} + +func (mc *mockCluster) MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { + return nil, nil +} + +func (mc *mockCluster) MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) { + return nil, nil +} + +func (mc *mockCluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) { + return nil, nil +} + +func (mc *mockCluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) { + return nil, nil +} + +func TestClientRejectOldCluster(t *testing.T) { + testutil.RegisterLeakDetection(t) + var tests = []struct { + name string + endpoints []string + versions []string + expectedError error + }{ + { + name: "all new versions with the same value", + endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, + versions: []string{"3.5.4", "3.5.4", "3.5.4"}, + expectedError: nil, + }, + { + name: "all new versions with different values", + endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, + versions: []string{"3.5.4", "3.5.4", "3.4.0"}, + expectedError: nil, + }, + { + name: "all old versions with different values", + endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, + versions: []string{"3.3.0", "3.3.0", "3.4.0"}, + expectedError: ErrOldCluster, + }, + { + name: "all old versions with the same value", + endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, + versions: []string{"3.3.0", "3.3.0", "3.3.0"}, + expectedError: ErrOldCluster, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if len(tt.endpoints) != len(tt.versions) || len(tt.endpoints) == 0 { + t.Errorf("Unexpected endpoints and versions length, len(endpoints):%d, len(versions):%d", len(tt.endpoints), len(tt.versions)) + return + } + endpointToVersion := make(map[string]string) + for j := range tt.endpoints { + endpointToVersion[tt.endpoints[j]] = tt.versions[j] + } + c := &Client{ + ctx: context.Background(), + cfg: Config{ + Endpoints: tt.endpoints, + }, + mu: new(sync.RWMutex), + Maintenance: &mockMaintenance{ + Version: endpointToVersion, + }, + } + + if err := c.checkVersion(); err != tt.expectedError { + t.Errorf("heckVersion err:%v", err) + } + }) + + } + +} + +type mockMaintenance struct { + Version map[string]string +} + +func (mm mockMaintenance) Status(ctx context.Context, endpoint string) (*StatusResponse, error) { + return &StatusResponse{Version: mm.Version[endpoint]}, nil +} + +func (mm mockMaintenance) AlarmList(ctx context.Context) (*AlarmResponse, error) { + return nil, nil +} + +func (mm mockMaintenance) AlarmDisarm(ctx context.Context, m *AlarmMember) (*AlarmResponse, error) { + return nil, nil +} + +func (mm mockMaintenance) Defragment(ctx context.Context, endpoint string) (*DefragmentResponse, error) { + return nil, nil +} + +func (mm mockMaintenance) HashKV(ctx context.Context, endpoint string, rev int64) (*HashKVResponse, error) { + return nil, nil +} + +func (mm mockMaintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) { + return nil, nil +} + +func (mm mockMaintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*MoveLeaderResponse, error) { + return nil, nil +} diff --git a/client/v3/cluster.go b/client/v3/cluster.go index 92d7cdb56b0f..1815c1c96408 100644 --- a/client/v3/cluster.go +++ b/client/v3/cluster.go @@ -93,7 +93,7 @@ func (c *cluster) memberAdd(ctx context.Context, peerAddrs []string, isLearner b } resp, err := c.remote.MemberAdd(ctx, r, c.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*MemberAddResponse)(resp), nil } @@ -102,7 +102,7 @@ func (c *cluster) MemberRemove(ctx context.Context, id uint64) (*MemberRemoveRes r := &pb.MemberRemoveRequest{ID: id} resp, err := c.remote.MemberRemove(ctx, r, c.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*MemberRemoveResponse)(resp), nil } @@ -119,7 +119,7 @@ func (c *cluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []strin if err == nil { return (*MemberUpdateResponse)(resp), nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } func (c *cluster) MemberList(ctx context.Context) (*MemberListResponse, error) { @@ -128,14 +128,14 @@ func (c *cluster) MemberList(ctx context.Context) (*MemberListResponse, error) { if err == nil { return (*MemberListResponse)(resp), nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } func (c *cluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) { r := &pb.MemberPromoteRequest{ID: id} resp, err := c.remote.MemberPromote(ctx, r, c.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*MemberPromoteResponse)(resp), nil } diff --git a/client/v3/config.go b/client/v3/config.go index 335a288732b5..6f5b41978d31 100644 --- a/client/v3/config.go +++ b/client/v3/config.go @@ -88,5 +88,14 @@ type Config struct { // PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs). PermitWithoutStream bool `json:"permit-without-stream"` + // MaxUnaryRetries is the maximum number of retries for unary RPCs. + MaxUnaryRetries uint `json:"max-unary-retries"` + + // BackoffWaitBetween is the wait time before retrying an RPC. + BackoffWaitBetween time.Duration `json:"backoff-wait-between"` + + // BackoffJitterFraction is the jitter fraction to randomize backoff wait time. + BackoffJitterFraction float64 `json:"backoff-jitter-fraction"` + // TODO: support custom balancer picker } diff --git a/client/v3/doc.go b/client/v3/doc.go index 645d744a5a7f..fd61aff117aa 100644 --- a/client/v3/doc.go +++ b/client/v3/doc.go @@ -61,7 +61,8 @@ // // 1. context error: canceled or deadline exceeded. // 2. gRPC error: e.g. when clock drifts in server-side before client's context deadline exceeded. -// See https://github.com/etcd-io/etcd/blob/main/api/v3rpc/rpctypes/error.go +// +// See https://github.com/etcd-io/etcd/blob/main/api/v3rpc/rpctypes/error.go // // Here is the example code to handle client errors: // @@ -102,5 +103,4 @@ // The grpc load balancer is registered statically and is shared across etcd clients. // To enable detailed load balancer logging, set the ETCD_CLIENT_DEBUG environment // variable. E.g. "ETCD_CLIENT_DEBUG=1". -// package clientv3 diff --git a/client/v3/experimental/recipes/barrier.go b/client/v3/experimental/recipes/barrier.go index 7e950a3e3854..ce88e4ff86d1 100644 --- a/client/v3/experimental/recipes/barrier.go +++ b/client/v3/experimental/recipes/barrier.go @@ -49,7 +49,7 @@ func (b *Barrier) Release() error { // Wait blocks on the barrier key until it is deleted. If there is no key, Wait // assumes Release has already been called and returns immediately. func (b *Barrier) Wait() error { - resp, err := b.client.Get(b.ctx, b.key, v3.WithFirstKey()...) + resp, err := b.client.Get(b.ctx, b.key) if err != nil { return err } diff --git a/client/v3/experimental/recipes/double_barrier.go b/client/v3/experimental/recipes/double_barrier.go index eac5d4f7fdbb..0fce8d0e7c0c 100644 --- a/client/v3/experimental/recipes/double_barrier.go +++ b/client/v3/experimental/recipes/double_barrier.go @@ -45,25 +45,46 @@ func NewDoubleBarrier(s *concurrency.Session, key string, count int) *DoubleBarr // Enter waits for "count" processes to enter the barrier then returns func (b *DoubleBarrier) Enter() error { client := b.s.Client() + + // Check the entered clients before creating the UniqueEphemeralKey, + // fail the request if there are already too many clients. + if resp1, err := b.enteredClients(client); err != nil { + return err + } else if len(resp1.Kvs) >= b.count { + return ErrTooManyClients + } + ek, err := newUniqueEphemeralKey(b.s, b.key+"/waiters") if err != nil { return err } b.myKey = ek - resp, err := client.Get(b.ctx, b.key+"/waiters", clientv3.WithPrefix()) + // Check the entered clients after creating the UniqueEphemeralKey + resp2, err := b.enteredClients(client) if err != nil { return err } + if len(resp2.Kvs) >= b.count { + lastWaiter := resp2.Kvs[b.count-1] + if ek.rev > lastWaiter.CreateRevision { + // delete itself now, otherwise other processes may need to wait + // until these keys are automatically deleted when the related + // lease expires. + if err = b.myKey.Delete(); err != nil { + // Nothing to do here. We have to wait for the key to be + // deleted when the lease expires. + } + return ErrTooManyClients + } - if len(resp.Kvs) > b.count { - return ErrTooManyClients - } - - if len(resp.Kvs) == b.count { - // unblock waiters - _, err = client.Put(b.ctx, b.key+"/ready", "") - return err + if ek.rev == lastWaiter.CreateRevision { + // TODO(ahrtr): we might need to compare ek.key and + // string(lastWaiter.Key), they should be equal. + // unblock all other waiters + _, err = client.Put(b.ctx, b.key+"/ready", "") + return err + } } _, err = WaitEvents( @@ -74,6 +95,18 @@ func (b *DoubleBarrier) Enter() error { return err } +// enteredClients gets all the entered clients, which are ordered by the +// createRevision in ascending order. +func (b *DoubleBarrier) enteredClients(cli *clientv3.Client) (*clientv3.GetResponse, error) { + resp, err := cli.Get(b.ctx, b.key+"/waiters", clientv3.WithPrefix(), + clientv3.WithSort(clientv3.SortByCreateRevision, clientv3.SortAscend)) + if err != nil { + return nil, err + } + + return resp, nil +} + // Leave waits for "count" processes to leave the barrier then returns func (b *DoubleBarrier) Leave() error { client := b.s.Client() @@ -96,7 +129,7 @@ func (b *DoubleBarrier) Leave() error { } isLowest := string(lowest.Key) == b.myKey.Key() - if len(resp.Kvs) == 1 { + if len(resp.Kvs) == 1 && isLowest { // this is the only node in the barrier; finish up if _, err = client.Delete(b.ctx, b.key+"/ready"); err != nil { return err diff --git a/client/v3/go.mod b/client/v3/go.mod index bf703a279750..cdd171be49ce 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -1,18 +1,43 @@ module go.etcd.io/etcd/client/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 require ( github.com/dustin/go-humanize v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/prometheus/client_golang v1.5.1 - go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 - go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 - google.golang.org/grpc v1.37.0 + github.com/prometheus/client_golang v1.11.1 + go.etcd.io/etcd/api/v3 v3.5.21 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 + go.uber.org/zap v1.17.0 + google.golang.org/grpc v1.59.0 sigs.k8s.io/yaml v1.2.0 ) +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + replace ( go.etcd.io/etcd/api/v3 => ../../api go.etcd.io/etcd/client/pkg/v3 => ../pkg diff --git a/client/v3/go.sum b/client/v3/go.sum index 6ac5bd1bd3c9..cf2983333535 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -1,79 +1,68 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.1 h1:7OO2CXWMYNDdaAzP51t4lCCZWwpQHmvPbm9sxWjm3So= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -87,141 +76,134 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 h1:040c3dLNhgFQkoojH2AMpHCy4SrvhmxdU72d9GLGGE0= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/client/v3/internal/endpoint/endpoint.go b/client/v3/internal/endpoint/endpoint.go index 1d3f1a7a2c7f..35a3fe8c3374 100644 --- a/client/v3/internal/endpoint/endpoint.go +++ b/client/v3/internal/endpoint/endpoint.go @@ -41,12 +41,8 @@ func extractHostFromHostPort(ep string) string { return host } -func extractHostFromPath(pathStr string) string { - return extractHostFromHostPort(path.Base(pathStr)) -} - -//mustSplit2 returns the values from strings.SplitN(s, sep, 2). -//If sep is not found, it returns ("", "", false) instead. +// mustSplit2 returns the values from strings.SplitN(s, sep, 2). +// If sep is not found, it returns ("", "", false) instead. func mustSplit2(s, sep string) (string, string) { spl := strings.SplitN(s, sep, 2) if len(spl) < 2 { @@ -81,11 +77,12 @@ func schemeToCredsRequirement(schema string) CredsRequirement { // The main differences: // - etcd supports unixs & https names as opposed to unix & http to // distinguish need to configure certificates. -// - etcd support http(s) names as opposed to tcp supported by grpc/dial method. -// - etcd supports unix(s)://local-file naming schema +// - etcd support http(s) names as opposed to tcp supported by grpc/dial method. +// - etcd supports unix(s)://local-file naming schema // (as opposed to unix:local-file canonical name used by grpc for current dir files). -// - Within the unix(s) schemas, the last segment (filename) without 'port' (content after colon) -// is considered serverName - to allow local testing of cert-protected communication. +// - Within the unix(s) schemas, the last segment (filename) without 'port' (content after colon) +// is considered serverName - to allow local testing of cert-protected communication. +// // See more: // - https://github.com/grpc/grpc-go/blob/26c143bd5f59344a4b8a1e491e0f5e18aa97abc7/internal/grpcutil/target.go#L47 // - https://golang.org/pkg/net/#Dial @@ -95,29 +92,29 @@ func translateEndpoint(ep string) (addr string, serverName string, requireCreds if strings.HasPrefix(ep, "unix:///") || strings.HasPrefix(ep, "unixs:///") { // absolute path case schema, absolutePath := mustSplit2(ep, "://") - return "unix://" + absolutePath, extractHostFromPath(absolutePath), schemeToCredsRequirement(schema) + return "unix://" + absolutePath, path.Base(absolutePath), schemeToCredsRequirement(schema) } if strings.HasPrefix(ep, "unix://") || strings.HasPrefix(ep, "unixs://") { // legacy etcd local path schema, localPath := mustSplit2(ep, "://") - return "unix:" + localPath, extractHostFromPath(localPath), schemeToCredsRequirement(schema) + return "unix:" + localPath, path.Base(localPath), schemeToCredsRequirement(schema) } schema, localPath := mustSplit2(ep, ":") - return "unix:" + localPath, extractHostFromPath(localPath), schemeToCredsRequirement(schema) + return "unix:" + localPath, path.Base(localPath), schemeToCredsRequirement(schema) } if strings.Contains(ep, "://") { url, err := url.Parse(ep) if err != nil { - return ep, extractHostFromHostPort(ep), CREDS_OPTIONAL + return ep, ep, CREDS_OPTIONAL } if url.Scheme == "http" || url.Scheme == "https" { - return url.Host, url.Hostname(), schemeToCredsRequirement(url.Scheme) + return url.Host, url.Host, schemeToCredsRequirement(url.Scheme) } - return ep, url.Hostname(), schemeToCredsRequirement(url.Scheme) + return ep, url.Host, schemeToCredsRequirement(url.Scheme) } // Handles plain addresses like 10.0.0.44:437. - return ep, extractHostFromHostPort(ep), CREDS_OPTIONAL + return ep, ep, CREDS_OPTIONAL } // RequiresCredentials returns whether given endpoint requires diff --git a/client/v3/internal/endpoint/endpoint_test.go b/client/v3/internal/endpoint/endpoint_test.go index bc6cd71399cc..7de9226ff19a 100644 --- a/client/v3/internal/endpoint/endpoint_test.go +++ b/client/v3/internal/endpoint/endpoint_test.go @@ -27,35 +27,35 @@ func Test_interpret(t *testing.T) { }{ {"127.0.0.1", "127.0.0.1", "127.0.0.1", CREDS_OPTIONAL}, {"localhost", "localhost", "localhost", CREDS_OPTIONAL}, - {"localhost:8080", "localhost:8080", "localhost", CREDS_OPTIONAL}, + {"localhost:8080", "localhost:8080", "localhost:8080", CREDS_OPTIONAL}, {"unix:127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CREDS_OPTIONAL}, - {"unix:127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1", CREDS_OPTIONAL}, + {"unix:127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CREDS_OPTIONAL}, {"unix://127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CREDS_OPTIONAL}, - {"unix://127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1", CREDS_OPTIONAL}, + {"unix://127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CREDS_OPTIONAL}, {"unixs:127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CREDS_REQUIRE}, - {"unixs:127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1", CREDS_REQUIRE}, + {"unixs:127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CREDS_REQUIRE}, {"unixs://127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CREDS_REQUIRE}, - {"unixs://127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1", CREDS_REQUIRE}, + {"unixs://127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CREDS_REQUIRE}, {"http://127.0.0.1", "127.0.0.1", "127.0.0.1", CREDS_DROP}, - {"http://127.0.0.1:8080", "127.0.0.1:8080", "127.0.0.1", CREDS_DROP}, + {"http://127.0.0.1:8080", "127.0.0.1:8080", "127.0.0.1:8080", CREDS_DROP}, {"https://127.0.0.1", "127.0.0.1", "127.0.0.1", CREDS_REQUIRE}, - {"https://127.0.0.1:8080", "127.0.0.1:8080", "127.0.0.1", CREDS_REQUIRE}, - {"https://localhost:20000", "localhost:20000", "localhost", CREDS_REQUIRE}, + {"https://127.0.0.1:8080", "127.0.0.1:8080", "127.0.0.1:8080", CREDS_REQUIRE}, + {"https://localhost:20000", "localhost:20000", "localhost:20000", CREDS_REQUIRE}, {"unix:///tmp/abc", "unix:///tmp/abc", "abc", CREDS_OPTIONAL}, {"unixs:///tmp/abc", "unix:///tmp/abc", "abc", CREDS_REQUIRE}, - {"unix:///tmp/abc:1234", "unix:///tmp/abc:1234", "abc", CREDS_OPTIONAL}, - {"unixs:///tmp/abc:1234", "unix:///tmp/abc:1234", "abc", CREDS_REQUIRE}, + {"unix:///tmp/abc:1234", "unix:///tmp/abc:1234", "abc:1234", CREDS_OPTIONAL}, + {"unixs:///tmp/abc:1234", "unix:///tmp/abc:1234", "abc:1234", CREDS_REQUIRE}, {"etcd.io", "etcd.io", "etcd.io", CREDS_OPTIONAL}, {"http://etcd.io/abc", "etcd.io", "etcd.io", CREDS_DROP}, {"dns://something-other", "dns://something-other", "something-other", CREDS_OPTIONAL}, - {"http://[2001:db8:1f70::999:de8:7648:6e8]:100/", "[2001:db8:1f70::999:de8:7648:6e8]:100", "2001:db8:1f70::999:de8:7648:6e8", CREDS_DROP}, - {"[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", "2001:db8:1f70::999:de8:7648:6e8", CREDS_OPTIONAL}, + {"http://[2001:db8:1f70::999:de8:7648:6e8]:100/", "[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", CREDS_DROP}, + {"[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", CREDS_OPTIONAL}, {"unix:unexpected-file_name#123$456", "unix:unexpected-file_name#123$456", "unexpected-file_name#123$456", CREDS_OPTIONAL}, } for _, tt := range tests { diff --git a/client/v3/kubernetes/client.go b/client/v3/kubernetes/client.go new file mode 100644 index 000000000000..11f2a456447e --- /dev/null +++ b/client/v3/kubernetes/client.go @@ -0,0 +1,136 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "context" + "fmt" + + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// New creates Client from config. +// Caller is responsible to call Close() to clean up client. +func New(cfg clientv3.Config) (*Client, error) { + c, err := clientv3.New(cfg) + if err != nil { + return nil, err + } + kc := &Client{ + Client: c, + } + kc.Kubernetes = kc + return kc, nil +} + +type Client struct { + *clientv3.Client + Kubernetes Interface +} + +var _ Interface = (*Client)(nil) + +func (k Client) Get(ctx context.Context, key string, opts GetOptions) (resp GetResponse, err error) { + rangeResp, err := k.KV.Get(ctx, key, clientv3.WithRev(opts.Revision), clientv3.WithLimit(1)) + if err != nil { + return resp, err + } + resp.Revision = rangeResp.Header.Revision + if len(rangeResp.Kvs) == 1 { + resp.KV = rangeResp.Kvs[0] + } + return resp, nil +} + +func (k Client) List(ctx context.Context, prefix string, opts ListOptions) (resp ListResponse, err error) { + rangeStart := prefix + if opts.Continue != "" { + rangeStart = opts.Continue + } + rangeEnd := clientv3.GetPrefixRangeEnd(prefix) + rangeResp, err := k.KV.Get(ctx, rangeStart, clientv3.WithRange(rangeEnd), clientv3.WithLimit(opts.Limit), clientv3.WithRev(opts.Revision)) + if err != nil { + return resp, err + } + resp.Kvs = rangeResp.Kvs + resp.Count = rangeResp.Count + resp.Revision = rangeResp.Header.Revision + return resp, nil +} + +func (k Client) Count(ctx context.Context, prefix string, _ CountOptions) (int64, error) { + resp, err := k.KV.Get(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + if err != nil { + return 0, err + } + return resp.Count, nil +} + +func (k Client) OptimisticPut(ctx context.Context, key string, value []byte, expectedRevision int64, opts PutOptions) (resp PutResponse, err error) { + txn := k.KV.Txn(ctx).If( + clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), + ).Then( + clientv3.OpPut(key, string(value), clientv3.WithLease(opts.LeaseID)), + ) + + if opts.GetOnFailure { + txn = txn.Else(clientv3.OpGet(key)) + } + + txnResp, err := txn.Commit() + if err != nil { + return resp, err + } + resp.Succeeded = txnResp.Succeeded + resp.Revision = txnResp.Header.Revision + if opts.GetOnFailure && !txnResp.Succeeded { + if len(txnResp.Responses) == 0 { + return resp, fmt.Errorf("invalid OptimisticPut response: %v", txnResp.Responses) + } + resp.KV = kvFromTxnResponse(txnResp.Responses[0]) + } + return resp, nil +} + +func (k Client) OptimisticDelete(ctx context.Context, key string, expectedRevision int64, opts DeleteOptions) (resp DeleteResponse, err error) { + txn := k.KV.Txn(ctx).If( + clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), + ).Then( + clientv3.OpDelete(key), + ) + if opts.GetOnFailure { + txn = txn.Else(clientv3.OpGet(key)) + } + txnResp, err := txn.Commit() + if err != nil { + return resp, err + } + resp.Succeeded = txnResp.Succeeded + resp.Revision = txnResp.Header.Revision + if opts.GetOnFailure && !txnResp.Succeeded { + resp.KV = kvFromTxnResponse(txnResp.Responses[0]) + } + return resp, nil +} + +func kvFromTxnResponse(resp *pb.ResponseOp) *mvccpb.KeyValue { + getResponse := resp.GetResponseRange() + if len(getResponse.Kvs) == 1 { + return getResponse.Kvs[0] + } + return nil +} diff --git a/client/v3/kubernetes/interface.go b/client/v3/kubernetes/interface.go new file mode 100644 index 000000000000..19b82a629270 --- /dev/null +++ b/client/v3/kubernetes/interface.go @@ -0,0 +1,140 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "context" + + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// Interface defines the minimal client-side interface that Kubernetes requires +// to interact with etcd. Methods below are standard etcd operations with +// semantics adjusted to better suit Kubernetes' needs. +type Interface interface { + // Get retrieves a single key-value pair from etcd. + // + // If opts.Revision is set to a non-zero value, the key-value pair is retrieved at the specified revision. + // If the required revision has been compacted, the request will fail with ErrCompacted. + Get(ctx context.Context, key string, opts GetOptions) (GetResponse, error) + + // List retrieves key-value pairs with the specified prefix, ordered lexicographically by key. + // + // If opts.Revision is non-zero, the key-value pairs are retrieved at the specified revision. + // If the required revision has been compacted, the request will fail with ErrCompacted. + // If opts.Limit is greater than zero, the number of returned key-value pairs is bounded by the limit. + // If opts.Continue is not empty, the listing will start from the key immediately after the one specified by Continue. + // The Continue value should be the last key returned in a previous paginated ListResponse. + List(ctx context.Context, prefix string, opts ListOptions) (ListResponse, error) + + // Count returns the number of keys with the specified prefix. + // + // Currently, there are no options for the Count operation. However, a placeholder options struct (CountOptions) + // is provided for future extensibility in case options become necessary. + Count(ctx context.Context, prefix string, opts CountOptions) (int64, error) + + // OptimisticPut creates or updates a key-value pair if the key has not been modified or created + // since the revision specified in expectedRevision. + // + // An OptimisticPut fails if the key has been modified since expectedRevision. + OptimisticPut(ctx context.Context, key string, value []byte, expectedRevision int64, opts PutOptions) (PutResponse, error) + + // OptimisticDelete deletes the key-value pair if it hasn't been modified since the revision + // specified in expectedRevision. + // + // An OptimisticDelete fails if the key has been modified since expectedRevision. + OptimisticDelete(ctx context.Context, key string, expectedRevision int64, opts DeleteOptions) (DeleteResponse, error) +} + +type GetOptions struct { + // Revision is the point-in-time of the etcd key-value store to use for the Get operation. + // If Revision is 0, it gets the latest value. + Revision int64 +} + +type ListOptions struct { + // Revision is the point-in-time of the etcd key-value store to use for the List operation. + // If Revision is 0, it gets the latest values. + Revision int64 + + // Limit is the maximum number of keys to return for a List operation. + // 0 means no limitation. + Limit int64 + + // Continue is a key from which to resume the List operation, excluding the given key. + // It should be set to the last key from a previous ListResponse when paginating. + Continue string +} + +// CountOptions is a placeholder for potential future options for the Count operation. +type CountOptions struct{} + +type PutOptions struct { + // GetOnFailure specifies whether to return the modified key-value pair if the Put operation fails due to a revision mismatch. + GetOnFailure bool + + // LeaseID is the ID of a lease to associate with the key allowing for automatic deletion after lease expires after it's TTL (time to live). + // Deprecated: Should be replaced with TTL when Interface starts using one lease per object. + LeaseID clientv3.LeaseID +} + +type DeleteOptions struct { + // GetOnFailure specifies whether to return the modified key-value pair if the Delete operation fails due to a revision mismatch. + GetOnFailure bool +} + +type GetResponse struct { + // KV is the key-value pair retrieved from etcd. + KV *mvccpb.KeyValue + + // Revision is the revision of the key-value store at the time of the Get operation. + Revision int64 +} + +type ListResponse struct { + // Kvs is the list of key-value pairs retrieved from etcd, ordered lexicographically by key. + Kvs []*mvccpb.KeyValue + + // Count is the total number of keys with the specified prefix, even if not all were returned due to a limit. + Count int64 + + // Revision is the revision of the key-value store at the time of the List operation. + Revision int64 +} + +type PutResponse struct { + // KV is the created or updated key-value pair. If the Put operation failed and GetOnFailure was true, this + // will be the modified key-value pair that caused the failure. + KV *mvccpb.KeyValue + + // Succeeded indicates whether the Put operation was successful. + Succeeded bool + + // Revision is the revision of the key-value store after the Put operation. + Revision int64 +} + +type DeleteResponse struct { + // KV is the deleted key-value pair. If the Delete operation failed and GetOnFailure was true, this + // will be the modified key-value pair that caused the failure. + KV *mvccpb.KeyValue + + // Succeeded indicates whether the Delete operation was successful. + Succeeded bool + + // Revision is the revision of the key-value store after the Delete operation. + Revision int64 +} diff --git a/client/v3/kv.go b/client/v3/kv.go index 5e9fb7d45896..be5b508dd615 100644 --- a/client/v3/kv.go +++ b/client/v3/kv.go @@ -112,23 +112,23 @@ func NewKVFromKVClient(remote pb.KVClient, c *Client) KV { func (kv *kv) Put(ctx context.Context, key, val string, opts ...OpOption) (*PutResponse, error) { r, err := kv.Do(ctx, OpPut(key, val, opts...)) - return r.put, toErr(ctx, err) + return r.put, ContextError(ctx, err) } func (kv *kv) Get(ctx context.Context, key string, opts ...OpOption) (*GetResponse, error) { r, err := kv.Do(ctx, OpGet(key, opts...)) - return r.get, toErr(ctx, err) + return r.get, ContextError(ctx, err) } func (kv *kv) Delete(ctx context.Context, key string, opts ...OpOption) (*DeleteResponse, error) { r, err := kv.Do(ctx, OpDelete(key, opts...)) - return r.del, toErr(ctx, err) + return r.del, ContextError(ctx, err) } func (kv *kv) Compact(ctx context.Context, rev int64, opts ...CompactOption) (*CompactResponse, error) { resp, err := kv.remote.Compact(ctx, OpCompact(rev, opts...).toRequest(), kv.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*CompactResponse)(resp), err } @@ -173,5 +173,5 @@ func (kv *kv) Do(ctx context.Context, op Op) (OpResponse, error) { default: panic("Unknown op") } - return OpResponse{}, toErr(ctx, err) + return OpResponse{}, ContextError(ctx, err) } diff --git a/client/v3/lease.go b/client/v3/lease.go index bd31e6b4a5b4..4877ee949626 100644 --- a/client/v3/lease.go +++ b/client/v3/lease.go @@ -223,7 +223,7 @@ func (l *lessor) Grant(ctx context.Context, ttl int64) (*LeaseGrantResponse, err } return gresp, nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } func (l *lessor) Revoke(ctx context.Context, id LeaseID) (*LeaseRevokeResponse, error) { @@ -232,14 +232,14 @@ func (l *lessor) Revoke(ctx context.Context, id LeaseID) (*LeaseRevokeResponse, if err == nil { return (*LeaseRevokeResponse)(resp), nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } func (l *lessor) TimeToLive(ctx context.Context, id LeaseID, opts ...LeaseOption) (*LeaseTimeToLiveResponse, error) { r := toLeaseTimeToLiveRequest(id, opts...) resp, err := l.remote.LeaseTimeToLive(ctx, r, l.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } gresp := &LeaseTimeToLiveResponse{ ResponseHeader: resp.GetHeader(), @@ -260,9 +260,15 @@ func (l *lessor) Leases(ctx context.Context) (*LeaseLeasesResponse, error) { } return &LeaseLeasesResponse{ResponseHeader: resp.GetHeader(), Leases: leases}, nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } +// To identify the context passed to `KeepAlive`, a key/value pair is +// attached to the context. The key is a `keepAliveCtxKey` object, and +// the value is the pointer to the context object itself, ensuring +// uniqueness as each context has a unique memory address. +type keepAliveCtxKey struct{} + func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error) { ch := make(chan *LeaseKeepAliveResponse, LeaseResponseChSize) @@ -277,6 +283,10 @@ func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAl default: } ka, ok := l.keepAlives[id] + + if ctx.Done() != nil { + ctx = context.WithValue(ctx, keepAliveCtxKey{}, &ctx) + } if !ok { // create fresh keep alive ka = &keepAlive{ @@ -294,7 +304,9 @@ func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAl } l.mu.Unlock() - go l.keepAliveCtxCloser(ctx, id, ka.donec) + if ctx.Done() != nil { + go l.keepAliveCtxCloser(ctx, id, ka.donec) + } l.firstKeepAliveOnce.Do(func() { go l.recvKeepAliveLoop() go l.deadlineLoop() @@ -313,7 +325,7 @@ func (l *lessor) KeepAliveOnce(ctx context.Context, id LeaseID) (*LeaseKeepAlive return resp, err } if isHaltErr(ctx, err) { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } } } @@ -345,7 +357,7 @@ func (l *lessor) keepAliveCtxCloser(ctx context.Context, id LeaseID, donec <-cha // close channel and remove context if still associated with keep alive for i, c := range ka.ctxs { - if c == ctx { + if c.Value(keepAliveCtxKey{}) == ctx.Value(keepAliveCtxKey{}) { close(ka.chs[i]) ka.ctxs = append(ka.ctxs[:i], ka.ctxs[i+1:]...) ka.chs = append(ka.chs[:i], ka.chs[i+1:]...) @@ -397,26 +409,35 @@ func (l *lessor) closeRequireLeader() { } } -func (l *lessor) keepAliveOnce(ctx context.Context, id LeaseID) (*LeaseKeepAliveResponse, error) { +func (l *lessor) keepAliveOnce(ctx context.Context, id LeaseID) (karesp *LeaseKeepAliveResponse, ferr error) { cctx, cancel := context.WithCancel(ctx) defer cancel() stream, err := l.remote.LeaseKeepAlive(cctx, l.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } + defer func() { + if err := stream.CloseSend(); err != nil { + if ferr == nil { + ferr = ContextError(ctx, err) + } + return + } + }() + err = stream.Send(&pb.LeaseKeepAliveRequest{ID: int64(id)}) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } resp, rerr := stream.Recv() if rerr != nil { - return nil, toErr(ctx, rerr) + return nil, ContextError(ctx, rerr) } - karesp := &LeaseKeepAliveResponse{ + karesp = &LeaseKeepAliveResponse{ ResponseHeader: resp.GetHeader(), ID: LeaseID(resp.ID), TTL: resp.TTL, @@ -450,7 +471,7 @@ func (l *lessor) recvKeepAliveLoop() (gerr error) { return err } - if toErr(l.stopCtx, err) == rpctypes.ErrNoLeader { + if ContextError(l.stopCtx, err) == rpctypes.ErrNoLeader { l.closeRequireLeader() } break diff --git a/client/v3/leasing/doc.go b/client/v3/leasing/doc.go index fc97fc882689..c38af3562b75 100644 --- a/client/v3/leasing/doc.go +++ b/client/v3/leasing/doc.go @@ -19,28 +19,27 @@ // // First, create a leasing KV from a clientv3.Client 'cli': // -// lkv, err := leasing.NewKV(cli, "leasing-prefix") -// if err != nil { -// // handle error -// } +// lkv, err := leasing.NewKV(cli, "leasing-prefix") +// if err != nil { +// // handle error +// } // // A range request for a key "abc" tries to acquire a leasing key so it can cache the range's // key locally. On the server, the leasing key is stored to "leasing-prefix/abc": // -// resp, err := lkv.Get(context.TODO(), "abc") +// resp, err := lkv.Get(context.TODO(), "abc") // // Future linearized read requests using 'lkv' will be served locally for the lease's lifetime: // -// resp, err = lkv.Get(context.TODO(), "abc") +// resp, err = lkv.Get(context.TODO(), "abc") // // If another leasing client writes to a leased key, then the owner relinquishes its exclusive // access, permitting the writer to modify the key: // -// lkv2, err := leasing.NewKV(cli, "leasing-prefix") -// if err != nil { -// // handle error -// } -// lkv2.Put(context.TODO(), "abc", "456") -// resp, err = lkv.Get("abc") -// +// lkv2, err := leasing.NewKV(cli, "leasing-prefix") +// if err != nil { +// // handle error +// } +// lkv2.Put(context.TODO(), "abc", "456") +// resp, err = lkv.Get("abc") package leasing diff --git a/client/v3/logger.go b/client/v3/logger.go index 0c019df6c528..eaa35f2d3acb 100644 --- a/client/v3/logger.go +++ b/client/v3/logger.go @@ -19,8 +19,8 @@ import ( "os" "go.etcd.io/etcd/client/pkg/v3/logutil" - "go.uber.org/zap" "go.uber.org/zap/zapcore" + "go.uber.org/zap/zapgrpc" "google.golang.org/grpc/grpclog" ) @@ -28,14 +28,12 @@ func init() { // We override grpc logger only when the environment variable is set // in order to not interfere by default with user's code or other libraries. if os.Getenv("ETCD_CLIENT_DEBUG") != "" { - // We don't use grpc_zap.ReplaceGrpcLoggerV2(lg) to not push (wide) set - // of grpc-ecosystem/go-grpc-middleware dependencies on etcd-client users. - lg, err := logutil.NewGRPCLoggerV2(createDefaultZapLoggerConfig()) + lg, err := logutil.CreateDefaultZapLogger(etcdClientDebugLevel()) if err != nil { panic(err) } - - grpclog.SetLoggerV2(lg) + lg = lg.Named("etcd-client") + grpclog.SetLoggerV2(zapgrpc.NewLogger(lg)) } } @@ -53,23 +51,9 @@ func etcdClientDebugLevel() zapcore.Level { return zapcore.InfoLevel } var l zapcore.Level - if err := l.Set(envLevel); err == nil { - log.Printf("Deprecated env ETCD_CLIENT_DEBUG value. Using default level: 'info'") + if err := l.Set(envLevel); err != nil { + log.Printf("Invalid value for environment variable 'ETCD_CLIENT_DEBUG'. Using default level: 'info'") return zapcore.InfoLevel } return l } - -func createDefaultZapLoggerConfig() zap.Config { - lcfg := logutil.DefaultZapLoggerConfig - lcfg.Level = zap.NewAtomicLevelAt(etcdClientDebugLevel()) - return lcfg -} - -func createDefaultZapLogger() (*zap.Logger, error) { - c, err := createDefaultZapLoggerConfig().Build() - if err != nil { - return nil, err - } - return c.Named("etcd-client"), nil -} diff --git a/client/v3/maintenance.go b/client/v3/maintenance.go index dbea530e66a2..71b28e6dc3f9 100644 --- a/client/v3/maintenance.go +++ b/client/v3/maintenance.go @@ -92,6 +92,7 @@ func NewMaintenance(c *Client) Maintenance { err = c.getToken(dctx) cancel() if err != nil { + conn.Close() return nil, nil, fmt.Errorf("failed to getToken from endpoint %s with maintenance client: %v", endpoint, err) } cancel = func() { conn.Close() } @@ -129,7 +130,7 @@ func (m *maintenance) AlarmList(ctx context.Context) (*AlarmResponse, error) { if err == nil { return (*AlarmResponse)(resp), nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } func (m *maintenance) AlarmDisarm(ctx context.Context, am *AlarmMember) (*AlarmResponse, error) { @@ -142,13 +143,13 @@ func (m *maintenance) AlarmDisarm(ctx context.Context, am *AlarmMember) (*AlarmR if req.MemberID == 0 && req.Alarm == pb.AlarmType_NONE { ar, err := m.AlarmList(ctx) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } ret := AlarmResponse{} for _, am := range ar.Alarms { dresp, derr := m.AlarmDisarm(ctx, (*AlarmMember)(am)) if derr != nil { - return nil, toErr(ctx, derr) + return nil, ContextError(ctx, derr) } ret.Alarms = append(ret.Alarms, dresp.Alarms...) } @@ -159,18 +160,18 @@ func (m *maintenance) AlarmDisarm(ctx context.Context, am *AlarmMember) (*AlarmR if err == nil { return (*AlarmResponse)(resp), nil } - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } func (m *maintenance) Defragment(ctx context.Context, endpoint string) (*DefragmentResponse, error) { remote, cancel, err := m.dial(endpoint) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } defer cancel() resp, err := remote.Defragment(ctx, &pb.DefragmentRequest{}, m.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*DefragmentResponse)(resp), nil } @@ -178,12 +179,12 @@ func (m *maintenance) Defragment(ctx context.Context, endpoint string) (*Defragm func (m *maintenance) Status(ctx context.Context, endpoint string) (*StatusResponse, error) { remote, cancel, err := m.dial(endpoint) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } defer cancel() resp, err := remote.Status(ctx, &pb.StatusRequest{}, m.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*StatusResponse)(resp), nil } @@ -192,12 +193,12 @@ func (m *maintenance) HashKV(ctx context.Context, endpoint string, rev int64) (* remote, cancel, err := m.dial(endpoint) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } defer cancel() resp, err := remote.HashKV(ctx, &pb.HashKVRequest{Revision: rev}, m.callOpts...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } return (*HashKVResponse)(resp), nil } @@ -205,7 +206,7 @@ func (m *maintenance) HashKV(ctx context.Context, endpoint string, rev int64) (* func (m *maintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) { ss, err := m.remote.Snapshot(ctx, &pb.SnapshotRequest{}, append(m.callOpts, withMax(defaultStreamMaxRetries))...) if err != nil { - return nil, toErr(ctx, err) + return nil, ContextError(ctx, err) } m.lg.Info("opened snapshot stream; downloading") @@ -245,10 +246,10 @@ type snapshotReadCloser struct { func (rc *snapshotReadCloser) Read(p []byte) (n int, err error) { n, err = rc.ReadCloser.Read(p) - return n, toErr(rc.ctx, err) + return n, ContextError(rc.ctx, err) } func (m *maintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*MoveLeaderResponse, error) { resp, err := m.remote.MoveLeader(ctx, &pb.MoveLeaderRequest{TargetID: transfereeID}, m.callOpts...) - return (*MoveLeaderResponse)(resp), toErr(ctx, err) + return (*MoveLeaderResponse)(resp), ContextError(ctx, err) } diff --git a/client/v3/mirror/syncer.go b/client/v3/mirror/syncer.go index c25ee9b0c555..8a9ad3faf125 100644 --- a/client/v3/mirror/syncer.go +++ b/client/v3/mirror/syncer.go @@ -18,7 +18,7 @@ package mirror import ( "context" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" ) const ( @@ -52,7 +52,13 @@ func (s *syncer) SyncBase(ctx context.Context) (<-chan clientv3.GetResponse, cha // if rev is not specified, we will choose the most recent revision. if s.rev == 0 { - resp, err := s.c.Get(ctx, "foo") + // If len(s.prefix) == 0, we will check a random key to fetch the most recent + // revision (foo), otherwise we use the provided prefix. + checkPath := "foo" + if len(s.prefix) != 0 { + checkPath = s.prefix + } + resp, err := s.c.Get(ctx, checkPath) if err != nil { errchan <- err close(respchan) diff --git a/client/v3/namespace/doc.go b/client/v3/namespace/doc.go index 01849b150abd..689e0e0bb382 100644 --- a/client/v3/namespace/doc.go +++ b/client/v3/namespace/doc.go @@ -39,5 +39,4 @@ // resp, _ = cli.Get(context.TODO(), "abc") // fmt.Printf("%s\n", resp.Kvs[0].Value) // // Output: 456 -// package namespace diff --git a/client/v3/naming/doc.go b/client/v3/naming/doc.go index 55130fa5b149..f2050a6aa6c2 100644 --- a/client/v3/naming/doc.go +++ b/client/v3/naming/doc.go @@ -13,10 +13,10 @@ // limitations under the License. // Package naming provides: -// - subpackage endpoints: an abstraction layer to store and read endpoints -// information from etcd. -// - subpackage resolver: an etcd-backed gRPC resolver for discovering gRPC -// services based on the endpoints configuration +// - subpackage endpoints: an abstraction layer to store and read endpoints +// information from etcd. +// - subpackage resolver: an etcd-backed gRPC resolver for discovering gRPC +// services based on the endpoints configuration // // To use, first import the packages: // @@ -55,5 +55,4 @@ // em := endpoints.NewManager(c, service) // return em.AddEndpoint(c.Ctx(), service+"/"+addr, endpoints.Endpoint{Addr:addr}, clientv3.WithLease(lid)); // } -// package naming diff --git a/client/v3/naming/endpoints/endpoints_impl.go b/client/v3/naming/endpoints/endpoints_impl.go index 37f04803e1e9..94cb73188b7d 100644 --- a/client/v3/naming/endpoints/endpoints_impl.go +++ b/client/v3/naming/endpoints/endpoints_impl.go @@ -78,7 +78,8 @@ func (m *endpointManager) DeleteEndpoint(ctx context.Context, key string, opts . } func (m *endpointManager) NewWatchChannel(ctx context.Context) (WatchChannel, error) { - resp, err := m.client.Get(ctx, m.target, clientv3.WithPrefix(), clientv3.WithSerializable()) + key := m.target + "/" + resp, err := m.client.Get(ctx, key, clientv3.WithPrefix(), clientv3.WithSerializable()) if err != nil { return nil, err } @@ -112,7 +113,8 @@ func (m *endpointManager) watch(ctx context.Context, rev int64, upch chan []*Upd lg := m.client.GetLogger() opts := []clientv3.OpOption{clientv3.WithRev(rev), clientv3.WithPrefix()} - wch := m.client.Watch(ctx, m.target, opts...) + key := m.target + "/" + wch := m.client.Watch(ctx, key, opts...) for { select { case <-ctx.Done(): @@ -157,7 +159,8 @@ func (m *endpointManager) watch(ctx context.Context, rev int64, upch chan []*Upd } func (m *endpointManager) List(ctx context.Context) (Key2EndpointMap, error) { - resp, err := m.client.Get(ctx, m.target, clientv3.WithPrefix(), clientv3.WithSerializable()) + key := m.target + "/" + resp, err := m.client.Get(ctx, key, clientv3.WithPrefix(), clientv3.WithSerializable()) if err != nil { return nil, err } diff --git a/client/v3/naming/resolver/resolver.go b/client/v3/naming/resolver/resolver.go index a44f61ae04cd..4c45ec9497fc 100644 --- a/client/v3/naming/resolver/resolver.go +++ b/client/v3/naming/resolver/resolver.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "strings" "sync" clientv3 "go.etcd.io/etcd/client/v3" @@ -17,9 +18,15 @@ type builder struct { } func (b builder) Build(target gresolver.Target, cc gresolver.ClientConn, opts gresolver.BuildOptions) (gresolver.Resolver, error) { + // Refer to https://github.com/grpc/grpc-go/blob/16d3df80f029f57cff5458f1d6da6aedbc23545d/clientconn.go#L1587-L1611 + endpoint := target.URL.Path + if endpoint == "" { + endpoint = target.URL.Opaque + } + endpoint = strings.TrimPrefix(endpoint, "/") r := &resolver{ c: b.c, - target: target.Endpoint, + target: endpoint, cc: cc, } r.ctx, r.cancel = context.WithCancel(context.Background()) diff --git a/client/v3/op.go b/client/v3/op.go index bd0f1f2f2136..5251906322c9 100644 --- a/client/v3/op.go +++ b/client/v3/op.go @@ -77,6 +77,9 @@ type Op struct { cmps []Cmp thenOps []Op elseOps []Op + + isOptsWithFromKey bool + isOptsWithPrefix bool } // accessors / mutators @@ -216,6 +219,10 @@ func (op Op) isWrite() bool { return op.t != tRange } +func NewOp() *Op { + return &Op{key: []byte("")} +} + // OpGet returns "get" operation based on given key and operation options. func OpGet(key string, opts ...OpOption) Op { // WithPrefix and WithFromKey are not supported together @@ -382,6 +389,7 @@ func getPrefix(key []byte) []byte { // can return 'foo1', 'foo2', and so on. func WithPrefix() OpOption { return func(op *Op) { + op.isOptsWithPrefix = true if len(op.key) == 0 { op.key, op.end = []byte{0}, []byte{0} return @@ -406,6 +414,7 @@ func WithFromKey() OpOption { op.key = []byte{0} } op.end = []byte("\x00") + op.isOptsWithFromKey = true } } @@ -554,7 +563,21 @@ func toLeaseTimeToLiveRequest(id LeaseID, opts ...LeaseOption) *pb.LeaseTimeToLi } // IsOptsWithPrefix returns true if WithPrefix option is called in the given opts. -func IsOptsWithPrefix(opts []OpOption) bool { return isOpFuncCalled("WithPrefix", opts) } +func IsOptsWithPrefix(opts []OpOption) bool { + ret := NewOp() + for _, opt := range opts { + opt(ret) + } + + return ret.isOptsWithPrefix +} // IsOptsWithFromKey returns true if WithFromKey option is called in the given opts. -func IsOptsWithFromKey(opts []OpOption) bool { return isOpFuncCalled("WithFromKey", opts) } +func IsOptsWithFromKey(opts []OpOption) bool { + ret := NewOp() + for _, opt := range opts { + opt(ret) + } + + return ret.isOptsWithFromKey +} diff --git a/client/v3/op_test.go b/client/v3/op_test.go index 762044fc5e21..7c964639cb35 100644 --- a/client/v3/op_test.go +++ b/client/v3/op_test.go @@ -36,3 +36,27 @@ func TestOpWithSort(t *testing.T) { t.Fatalf("expected %+v, got %+v", wreq, req) } } + +func TestIsOptsWithPrefix(t *testing.T) { + optswithprefix := []OpOption{WithPrefix()} + if !IsOptsWithPrefix(optswithprefix) { + t.Errorf("IsOptsWithPrefix = false, expected true") + } + + optswithfromkey := []OpOption{WithFromKey()} + if IsOptsWithPrefix(optswithfromkey) { + t.Errorf("IsOptsWithPrefix = true, expected false") + } +} + +func TestIsOptsWithFromKey(t *testing.T) { + optswithfromkey := []OpOption{WithFromKey()} + if !IsOptsWithFromKey(optswithfromkey) { + t.Errorf("IsOptsWithFromKey = false, expected true") + } + + optswithprefix := []OpOption{WithPrefix()} + if IsOptsWithFromKey(optswithprefix) { + t.Errorf("IsOptsWithFromKey = true, expected false") + } +} diff --git a/client/v3/ordering/doc.go b/client/v3/ordering/doc.go index 856f33058011..03588248bd6c 100644 --- a/client/v3/ordering/doc.go +++ b/client/v3/ordering/doc.go @@ -38,5 +38,4 @@ // cli.KV = ordering.NewKV(cli.KV, vf) // // Now calls using 'cli' will reject order violations with an error. -// package ordering diff --git a/client/v3/patch_cluster.go b/client/v3/patch_cluster.go new file mode 100644 index 000000000000..a6311746cc23 --- /dev/null +++ b/client/v3/patch_cluster.go @@ -0,0 +1,24 @@ +package clientv3 + +import ( + "context" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" +) + +// NonLinearizeableMemberLister is used by the discover-etcd-initial-cluster command to get a list of members to ensure that *this* +// member has been added to the list. This is needed on restart scenarios when there isn't quorum. We need the first +// two etcd servers to start without quorum having been established by finding themselves in the member list and moving +// past the gate. +type NonLinearizeableMemberLister interface { + // NonLinearizeableMemberList is like MemberList only without linearization. + NonLinearizeableMemberList(ctx context.Context) (*MemberListResponse, error) +} + +func (c *cluster) NonLinearizeableMemberList(ctx context.Context) (*MemberListResponse, error) { + // it is safe to retry on list. + resp, err := c.remote.MemberList(ctx, &pb.MemberListRequest{}, c.callOpts...) + if err == nil { + return (*MemberListResponse)(resp), nil + } + return nil, ContextError(ctx, err) +} diff --git a/client/v3/retry_interceptor.go b/client/v3/retry_interceptor.go index 9586c334a3d5..8c50dcfa93d7 100644 --- a/client/v3/retry_interceptor.go +++ b/client/v3/retry_interceptor.go @@ -19,6 +19,7 @@ package clientv3 import ( "context" + "errors" "io" "sync" "time" @@ -73,14 +74,8 @@ func (c *Client) unaryClientInterceptor(optFuncs ...retryOption) grpc.UnaryClien // its the callCtx deadline or cancellation, in which case try again. continue } - if callOpts.retryAuth && rpctypes.Error(lastErr) == rpctypes.ErrInvalidAuthToken { - // clear auth token before refreshing it. - // call c.Auth.Authenticate with an invalid token will always fail the auth check on the server-side, - // if the server has not apply the patch of pr #12165 (https://github.com/etcd-io/etcd/pull/12165) - // and a rpctypes.ErrInvalidAuthToken will recursively call c.getToken until system run out of resource. - c.authTokenBundle.UpdateAuthToken("") - - gterr := c.getToken(ctx) + if c.shouldRefreshToken(lastErr, callOpts) { + gterr := c.refreshToken(ctx) if gterr != nil { c.GetLogger().Warn( "retrying of unary invoker failed to fetch new auth token", @@ -91,7 +86,7 @@ func (c *Client) unaryClientInterceptor(optFuncs ...retryOption) grpc.UnaryClien } continue } - if !isSafeRetry(c.lg, lastErr, callOpts) { + if !isSafeRetry(c, lastErr, callOpts) { return lastErr } } @@ -148,6 +143,37 @@ func (c *Client) streamClientInterceptor(optFuncs ...retryOption) grpc.StreamCli } } +// shouldRefreshToken checks whether there's a need to refresh the token based on the error and callOptions, +// and returns a boolean value. +func (c *Client) shouldRefreshToken(err error, callOpts *options) bool { + if rpctypes.Error(err) == rpctypes.ErrUserEmpty { + // refresh the token when username, password is present but the server returns ErrUserEmpty + // which is possible when the client token is cleared somehow + return c.authTokenBundle != nil // equal to c.Username != "" && c.Password != "" + } + + return callOpts.retryAuth && + (rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken || rpctypes.Error(err) == rpctypes.ErrAuthOldRevision) +} + +func (c *Client) refreshToken(ctx context.Context) error { + if c.authTokenBundle == nil { + // c.authTokenBundle will be initialized only when + // c.Username != "" && c.Password != "". + // + // When users use the TLS CommonName based authentication, the + // authTokenBundle is always nil. But it's possible for the clients + // to get `rpctypes.ErrAuthOldRevision` response when the clients + // concurrently modify auth data (e.g, addUser, deleteUser etc.). + // In this case, there is no need to refresh the token; instead the + // clients just need to retry the operations (e.g. Put, Delete etc). + return nil + } + // clear auth token before refreshing it. + c.authTokenBundle.UpdateAuthToken("") + return c.getToken(ctx) +} + // type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a // proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish // a new ClientStream according to the retry policy. @@ -245,11 +271,8 @@ func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{} // its the callCtx deadline or cancellation, in which case try again. return true, err } - if s.callOpts.retryAuth && rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken { - // clear auth token to avoid failure when call getToken - s.client.authTokenBundle.UpdateAuthToken("") - - gterr := s.client.getToken(s.ctx) + if s.client.shouldRefreshToken(err, s.callOpts) { + gterr := s.client.refreshToken(s.ctx) if gterr != nil { s.client.lg.Warn("retry failed to fetch new auth token", zap.Error(gterr)) return false, err // return the original error for simplicity @@ -257,7 +280,7 @@ func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{} return true, err } - return isSafeRetry(s.client.lg, err, s.callOpts), err + return isSafeRetry(s.client, err, s.callOpts), err } func (s *serverStreamingRetryingStream) reestablishStreamAndResendBuffer(callCtx context.Context) (grpc.ClientStream, error) { @@ -297,17 +320,28 @@ func waitRetryBackoff(ctx context.Context, attempt uint, callOpts *options) erro } // isSafeRetry returns "true", if request is safe for retry with the given error. -func isSafeRetry(lg *zap.Logger, err error, callOpts *options) bool { +func isSafeRetry(c *Client, err error, callOpts *options) bool { if isContextError(err) { return false } + + // Situation when learner refuses RPC it is supposed to not serve is from the server + // perspective not retryable. + // But for backward-compatibility reasons we need to support situation that + // customer provides mix of learners (not yet voters) and voters with an + // expectation to pick voter in the next attempt. + // TODO: Ideally client should be 'aware' which endpoint represents: leader/voter/learner with high probability. + if errors.Is(err, rpctypes.ErrGPRCNotSupportedForLearner) && len(c.Endpoints()) > 1 { + return true + } + switch callOpts.retryPolicy { case repeatable: return isSafeRetryImmutableRPC(err) case nonRepeatable: return isSafeRetryMutableRPC(err) default: - lg.Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String())) + c.lg.Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String())) return false } } diff --git a/client/v3/retry_interceptor_test.go b/client/v3/retry_interceptor_test.go new file mode 100644 index 000000000000..b850b56e0f5d --- /dev/null +++ b/client/v3/retry_interceptor_test.go @@ -0,0 +1,124 @@ +package clientv3 + +import ( + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "go.etcd.io/etcd/client/v3/credentials" + grpccredentials "google.golang.org/grpc/credentials" + "testing" +) + +type dummyAuthTokenBundle struct{} + +func (d dummyAuthTokenBundle) TransportCredentials() grpccredentials.TransportCredentials { + return nil +} + +func (d dummyAuthTokenBundle) PerRPCCredentials() grpccredentials.PerRPCCredentials { + return nil +} + +func (d dummyAuthTokenBundle) NewWithMode(mode string) (grpccredentials.Bundle, error) { + return nil, nil +} + +func (d dummyAuthTokenBundle) UpdateAuthToken(token string) { +} + +func TestClientShouldRefreshToken(t *testing.T) { + type fields struct { + authTokenBundle credentials.Bundle + } + type args struct { + err error + callOpts *options + } + + optsWithTrue := &options{ + retryAuth: true, + } + optsWithFalse := &options{ + retryAuth: false, + } + + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "ErrUserEmpty and non nil authTokenBundle", + fields: fields{ + authTokenBundle: &dummyAuthTokenBundle{}, + }, + args: args{rpctypes.ErrGRPCUserEmpty, optsWithTrue}, + want: true, + }, + { + name: "ErrUserEmpty and nil authTokenBundle", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCUserEmpty, optsWithTrue}, + want: false, + }, + { + name: "ErrGRPCInvalidAuthToken and retryAuth", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithTrue}, + want: true, + }, + { + name: "ErrGRPCInvalidAuthToken and !retryAuth", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithFalse}, + want: false, + }, + { + name: "ErrGRPCAuthOldRevision and retryAuth", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCAuthOldRevision, optsWithTrue}, + want: true, + }, + { + name: "ErrGRPCAuthOldRevision and !retryAuth", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCAuthOldRevision, optsWithFalse}, + want: false, + }, + { + name: "Other error and retryAuth", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCAuthFailed, optsWithTrue}, + want: false, + }, + { + name: "Other error and !retryAuth", + fields: fields{ + authTokenBundle: nil, + }, + args: args{rpctypes.ErrGRPCAuthFailed, optsWithFalse}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Client{ + authTokenBundle: tt.fields.authTokenBundle, + } + if got := c.shouldRefreshToken(tt.args.err, tt.args.callOpts); got != tt.want { + t.Errorf("shouldRefreshToken() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index b6eb592dda3d..39d5211aae13 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -44,9 +44,6 @@ func hasChecksum(n int64) bool { // selected node, and saved snapshot is the point-in-time state of // the selected node. func Save(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, dbPath string) error { - if lg == nil { - lg = zap.NewExample() - } cfg.Logger = lg.Named("client") if len(cfg.Endpoints) != 1 { return fmt.Errorf("snapshot must be requested to one selected node, not multiple %v", cfg.Endpoints) diff --git a/client/v3/txn.go b/client/v3/txn.go index 22301fba6b14..e31bfe0b94d2 100644 --- a/client/v3/txn.go +++ b/client/v3/txn.go @@ -25,15 +25,14 @@ import ( // Txn is the interface that wraps mini-transactions. // -// Txn(context.TODO()).If( -// Compare(Value(k1), ">", v1), -// Compare(Version(k1), "=", 2) -// ).Then( -// OpPut(k2,v2), OpPut(k3,v3) -// ).Else( -// OpPut(k4,v4), OpPut(k5,v5) -// ).Commit() -// +// Txn(context.TODO()).If( +// Compare(Value(k1), ">", v1), +// Compare(Version(k1), "=", 2) +// ).Then( +// OpPut(k2,v2), OpPut(k3,v3) +// ).Else( +// OpPut(k4,v4), OpPut(k5,v5) +// ).Commit() type Txn interface { // If takes a list of comparison. If all comparisons passed in succeed, // the operations passed into Then() will be executed. Or the operations @@ -145,7 +144,7 @@ func (txn *txn) Commit() (*TxnResponse, error) { var err error resp, err = txn.kv.remote.Txn(txn.ctx, r, txn.callOpts...) if err != nil { - return nil, toErr(txn.ctx, err) + return nil, ContextError(txn.ctx, err) } return (*TxnResponse)(resp), nil } diff --git a/client/v3/txn_test.go b/client/v3/txn_test.go index 4be01001368c..012c965a5121 100644 --- a/client/v3/txn_test.go +++ b/client/v3/txn_test.go @@ -23,7 +23,7 @@ import ( ) func TestTxnPanics(t *testing.T) { - testutil.BeforeTest(t) + testutil.RegisterLeakDetection(t) kv := &kv{} diff --git a/client/v3/utils.go b/client/v3/utils.go index b998c41b90fb..850275877d32 100644 --- a/client/v3/utils.go +++ b/client/v3/utils.go @@ -16,9 +16,6 @@ package clientv3 import ( "math/rand" - "reflect" - "runtime" - "strings" "time" ) @@ -32,18 +29,3 @@ func jitterUp(duration time.Duration, jitter float64) time.Duration { multiplier := jitter * (rand.Float64()*2 - 1) return time.Duration(float64(duration) * (1 + multiplier)) } - -// Check if the provided function is being called in the op options. -func isOpFuncCalled(op string, opts []OpOption) bool { - for _, opt := range opts { - v := reflect.ValueOf(opt) - if v.Kind() == reflect.Func { - if opFunc := runtime.FuncForPC(v.Pointer()); opFunc != nil { - if strings.Contains(opFunc.Name(), op) { - return true - } - } - } - } - return false -} diff --git a/client/v3/watch.go b/client/v3/watch.go index b73925ba128a..725e8a869e39 100644 --- a/client/v3/watch.go +++ b/client/v3/watch.go @@ -37,6 +37,13 @@ const ( EventTypePut = mvccpb.PUT closeSendErrTimeout = 250 * time.Millisecond + + // AutoWatchID is the watcher ID passed in WatchStream.Watch when no + // user-provided ID is available. If pass, an ID will automatically be assigned. + AutoWatchID = 0 + + // InvalidWatchID represents an invalid watch ID and prevents duplication with an existing watch. + InvalidWatchID = -1 ) type Event mvccpb.Event @@ -435,7 +442,7 @@ func (w *watchGrpcStream) close() (err error) { case err = <-w.errc: default: } - return toErr(w.ctx, err) + return ContextError(w.ctx, err) } func (w *watcher) closeStream(wgs *watchGrpcStream) { @@ -450,7 +457,7 @@ func (w *watcher) closeStream(wgs *watchGrpcStream) { func (w *watchGrpcStream) addSubstream(resp *pb.WatchResponse, ws *watcherStream) { // check watch ID for backward compatibility (<= v3.3) - if resp.WatchId == -1 || (resp.Canceled && resp.CancelReason != "") { + if resp.WatchId == InvalidWatchID || (resp.Canceled && resp.CancelReason != "") { w.closeErr = v3rpc.Error(errors.New(resp.CancelReason)) // failed; no channel close(ws.recvc) @@ -481,7 +488,7 @@ func (w *watchGrpcStream) closeSubstream(ws *watcherStream) { } else if ws.outc != nil { close(ws.outc) } - if ws.id != -1 { + if ws.id != InvalidWatchID { delete(w.substreams, ws.id) return } @@ -533,6 +540,7 @@ func (w *watchGrpcStream) run() { cancelSet := make(map[int64]struct{}) var cur *pb.WatchResponse + backoff := time.Millisecond for { select { // Watch() requested @@ -543,7 +551,7 @@ func (w *watchGrpcStream) run() { // TODO: pass custom watch ID? ws := &watcherStream{ initReq: *wreq, - id: -1, + id: InvalidWatchID, outc: outc, // unbuffered so resumes won't cause repeat events recvc: make(chan *WatchResponse), @@ -645,10 +653,11 @@ func (w *watchGrpcStream) run() { // watch client failed on Recv; spawn another if possible case err := <-w.errc: - if isHaltErr(w.ctx, err) || toErr(w.ctx, err) == v3rpc.ErrNoLeader { + if isHaltErr(w.ctx, err) || ContextError(w.ctx, err) == v3rpc.ErrNoLeader { closeErr = err return } + backoff = w.backoffIfUnavailable(backoff, err) if wc, closeErr = w.newWatchClient(); closeErr != nil { return } @@ -669,7 +678,7 @@ func (w *watchGrpcStream) run() { if len(w.substreams)+len(w.resuming) == 0 { return } - if ws.id != -1 { + if ws.id != InvalidWatchID { // client is closing an established watch; close it on the server proactively instead of waiting // to close when the next message arrives cancelSet[ws.id] = struct{}{} @@ -716,9 +725,9 @@ func (w *watchGrpcStream) dispatchEvent(pbresp *pb.WatchResponse) bool { cancelReason: pbresp.CancelReason, } - // watch IDs are zero indexed, so request notify watch responses are assigned a watch ID of -1 to + // watch IDs are zero indexed, so request notify watch responses are assigned a watch ID of InvalidWatchID to // indicate they should be broadcast. - if wr.IsProgressNotify() && pbresp.WatchId == -1 { + if wr.IsProgressNotify() && pbresp.WatchId == InvalidWatchID { return w.broadcastResponse(wr) } @@ -839,7 +848,7 @@ func (w *watchGrpcStream) serveSubstream(ws *watcherStream, resumec chan struct{ } } else { // current progress of watch; <= store revision - nextRev = wr.Header.Revision + nextRev = wr.Header.Revision + 1 } if len(wr.Events) > 0 { @@ -873,7 +882,7 @@ func (w *watchGrpcStream) newWatchClient() (pb.Watch_WatchClient, error) { w.resumec = make(chan struct{}) w.joinSubstreams() for _, ws := range w.substreams { - ws.id = -1 + ws.id = InvalidWatchID w.resuming = append(w.resuming, ws) } // strip out nils, if any @@ -963,6 +972,21 @@ func (w *watchGrpcStream) joinSubstreams() { var maxBackoff = 100 * time.Millisecond +func (w *watchGrpcStream) backoffIfUnavailable(backoff time.Duration, err error) time.Duration { + if isUnavailableErr(w.ctx, err) { + // retry, but backoff + if backoff < maxBackoff { + // 25% backoff factor + backoff = backoff + backoff/4 + if backoff > maxBackoff { + backoff = maxBackoff + } + } + time.Sleep(backoff) + } + return backoff +} + // openWatchClient retries opening a watch client until success or halt. // manually retry in case "ws==nil && err==nil" // TODO: remove FailFast=false @@ -983,17 +1007,7 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error) if isHaltErr(w.ctx, err) { return nil, v3rpc.Error(err) } - if isUnavailableErr(w.ctx, err) { - // retry, but backoff - if backoff < maxBackoff { - // 25% backoff factor - backoff = backoff + backoff/4 - if backoff > maxBackoff { - backoff = maxBackoff - } - } - time.Sleep(backoff) - } + backoff = w.backoffIfUnavailable(backoff, err) } return ws, nil } @@ -1022,7 +1036,7 @@ func (pr *progressRequest) toPB() *pb.WatchRequest { func streamKeyFromCtx(ctx context.Context) string { if md, ok := metadata.FromOutgoingContext(ctx); ok { - return fmt.Sprintf("%+v", md) + return fmt.Sprintf("%+v", map[string][]string(md)) } return "" } diff --git a/client/v3/watch_test.go b/client/v3/watch_test.go index 2a56ca4a9383..721fc4a8a237 100644 --- a/client/v3/watch_test.go +++ b/client/v3/watch_test.go @@ -15,8 +15,11 @@ package clientv3 import ( + "context" "testing" + "google.golang.org/grpc/metadata" + "go.etcd.io/etcd/api/v3/mvccpb" ) @@ -53,3 +56,53 @@ func TestEvent(t *testing.T) { } } } + +// TestStreamKeyFromCtx tests the streamKeyFromCtx function to ensure it correctly +// formats metadata as a map[string][]string when extracting metadata from the context. +// +// The fmt package in Go guarantees that maps are printed in a consistent order, +// sorted by the keys. This test verifies that the streamKeyFromCtx function +// produces the expected formatted string representation of metadata maps when called with +// various context scenarios. +func TestStreamKeyFromCtx(t *testing.T) { + tests := []struct { + name string + ctx context.Context + expected string + }{ + { + name: "multiple keys", + ctx: metadata.NewOutgoingContext(context.Background(), metadata.MD{ + "key1": []string{"value1"}, + "key2": []string{"value2a", "value2b"}, + }), + expected: "map[key1:[value1] key2:[value2a value2b]]", + }, + { + name: "no keys", + ctx: metadata.NewOutgoingContext(context.Background(), metadata.MD{}), + expected: "map[]", + }, + { + name: "only one key", + ctx: metadata.NewOutgoingContext(context.Background(), metadata.MD{ + "key1": []string{"value1", "value1a"}, + }), + expected: "map[key1:[value1 value1a]]", + }, + { + name: "no metadata", + ctx: context.Background(), + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := streamKeyFromCtx(tt.ctx) + if actual != tt.expected { + t.Errorf("streamKeyFromCtx() = %v, expected %v", actual, tt.expected) + } + }) + } +} diff --git a/cmd/etcd-tests-ext/main.go b/cmd/etcd-tests-ext/main.go new file mode 100644 index 000000000000..5f681ba832c6 --- /dev/null +++ b/cmd/etcd-tests-ext/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + "github.com/spf13/cobra" + + // The import below is necessary to ensure that the openshift etcd tests are registered with the extension. + _ "go.etcd.io/etcd/tests/v3/extended" +) + +func main() { + registry := e.NewRegistry() + ext := e.NewExtension("openshift", "payload", "etcd") + + // Suite: conformance/parallel (fast, parallel-safe) + ext.AddSuite(e.Suite{ + Name: "openshift/etcd/conformance/parallel", + Parents: []string{"openshift/conformance/parallel"}, + Qualifiers: []string{ + `!(name.contains("[Serial]") || name.contains("[Slow]"))`, + }, + }) + + // Suite: conformance/serial (explicitly serial tests) + ext.AddSuite(e.Suite{ + Name: "openshift/etcd/conformance/serial", + Parents: []string{"openshift/conformance/serial"}, + Qualifiers: []string{ + `name.contains("[Serial]")`, + }, + }) + + // Suite: optional/slow (long-running tests) + ext.AddSuite(e.Suite{ + Name: "openshift/etcd/optional/slow", + Parents: []string{"openshift/optional/slow"}, + Qualifiers: []string{ + `name.contains("[Slow]")`, + }, + }) + + // Suite: all (includes everything) + ext.AddSuite(e.Suite{ + Name: "openshift/etcd/all", + }) + + specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() + if err != nil { + panic(fmt.Sprintf("couldn't build extension test specs from ginkgo: %+v", err.Error())) + } + + // Ensure [Disruptive] tests are also [Serial] (for any future tests that might need this) + specs = specs.Walk(func(spec *et.ExtensionTestSpec) { + if strings.Contains(spec.Name, "[Disruptive]") && !strings.Contains(spec.Name, "[Serial]") { + spec.Name = strings.ReplaceAll( + spec.Name, + "[Disruptive]", + "[Serial][Disruptive]", + ) + } + }) + + // Preserve original-name labels for renamed tests + specs = specs.Walk(func(spec *et.ExtensionTestSpec) { + for label := range spec.Labels { + if strings.HasPrefix(label, "original-name:") { + parts := strings.SplitN(label, "original-name:", 2) + if len(parts) > 1 { + spec.OriginalName = parts[1] + } + } + } + }) + + // Ignore obsolete tests + ext.IgnoreObsoleteTests( + // "[sig-etcd] ", + ) + + // Initialize environment before running any tests + specs.AddBeforeAll(func() { + // do stuff + }) + + ext.AddSpecs(specs) + registry.Register(ext) + + root := &cobra.Command{ + Long: "Etcd Tests Extension", + } + + root.AddCommand(cmd.DefaultExtensionCommands(registry)...) + + if err := root.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/etcd.conf.yml.sample b/etcd.conf.yml.sample index 0d7a2c6b3d17..423a3fe42a2a 100644 --- a/etcd.conf.yml.sample +++ b/etcd.conf.yml.sample @@ -125,6 +125,15 @@ peer-transport-security: # Peer TLS using generated certificates. auto-tls: false + # Allowed CN for inter peer authentication. + allowed-cn: + + # Allowed TLS hostname for inter peer authentication. + allowed-hostname: + +# The validity period of the self-signed certificate, the unit is year. +self-signed-cert-validity: 1 + # Enable debug-level logging for etcd. log-level: debug diff --git a/etcdctl/ctlv3/command/check.go b/etcdctl/ctlv3/command/check.go index a2a5ca31593b..b7f9d750f2c7 100644 --- a/etcdctl/ctlv3/command/check.go +++ b/etcdctl/ctlv3/command/check.go @@ -107,13 +107,10 @@ var checkDatascaleCfgMap = map[string]checkDatascaleCfg{ // NewCheckCommand returns the cobra command for "check". func NewCheckCommand() *cobra.Command { cc := &cobra.Command{ - Use: "check ", - Short: "commands for checking properties of the etcd cluster", + Use: "check is no longer supported in OpenShift. Performance analysis should be performed using metrics, please see the etcd dashboards", + Short: "command no longer supported in OpenShift", } - cc.AddCommand(NewCheckPerfCommand()) - cc.AddCommand(NewCheckDatascaleCommand()) - return cc } diff --git a/etcdctl/ctlv3/command/defrag_command.go b/etcdctl/ctlv3/command/defrag_command.go index 42e47cbb9054..6876cd7881dc 100644 --- a/etcdctl/ctlv3/command/defrag_command.go +++ b/etcdctl/ctlv3/command/defrag_command.go @@ -46,6 +46,7 @@ func defragCommandFunc(cmd *cobra.Command, args []string) { if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } + return } failures := 0 diff --git a/etcdctl/ctlv3/command/del_command.go b/etcdctl/ctlv3/command/del_command.go index dbfd186dbad8..fefa9abd5b08 100644 --- a/etcdctl/ctlv3/command/del_command.go +++ b/etcdctl/ctlv3/command/del_command.go @@ -16,9 +16,10 @@ package command import ( "fmt" + "os" "github.com/spf13/cobra" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) @@ -26,6 +27,7 @@ var ( delPrefix bool delPrevKV bool delFromKey bool + delRange bool ) // NewDelCommand returns the cobra command for "del". @@ -39,6 +41,7 @@ func NewDelCommand() *cobra.Command { cmd.Flags().BoolVar(&delPrefix, "prefix", false, "delete keys with matching prefix") cmd.Flags().BoolVar(&delPrevKV, "prev-kv", false, "return deleted key-value pairs") cmd.Flags().BoolVar(&delFromKey, "from-key", false, "delete keys that are greater than or equal to the given key using byte compare") + cmd.Flags().BoolVar(&delRange, "range", false, "delete range of keys") return cmd } @@ -70,6 +73,9 @@ func getDelOp(args []string) (string, []clientv3.OpOption) { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("too many arguments, only accept one argument when `--prefix` or `--from-key` is set")) } opts = append(opts, clientv3.WithRange(args[1])) + if !delRange { + fmt.Fprintln(os.Stderr, "In etcd v3.6, the operation will be suspended for a few seconds to provide the user time to verify range.") + } } if delPrefix { diff --git a/etcdctl/ctlv3/command/ep_command.go b/etcdctl/ctlv3/command/ep_command.go index 9e4aad04d8f5..5de3a990d939 100644 --- a/etcdctl/ctlv3/command/ep_command.go +++ b/etcdctl/ctlv3/command/ep_command.go @@ -22,6 +22,7 @@ import ( "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "go.etcd.io/etcd/client/pkg/v3/logutil" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/pkg/v3/flags" @@ -88,7 +89,7 @@ type epHealth struct { // epHealthCommandFunc executes the "endpoint-health" command. func epHealthCommandFunc(cmd *cobra.Command, args []string) { - lg, err := zap.NewProduction() + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } diff --git a/etcdctl/ctlv3/command/global.go b/etcdctl/ctlv3/command/global.go index c50ab5963a51..c12106b49458 100644 --- a/etcdctl/ctlv3/command/global.go +++ b/etcdctl/ctlv3/command/global.go @@ -25,6 +25,7 @@ import ( "time" "github.com/bgentry/speakeasy" + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/v3" @@ -114,7 +115,7 @@ func (*discardValue) Set(string) error { return nil } func (*discardValue) Type() string { return "" } func clientConfigFromCmd(cmd *cobra.Command) *clientConfig { - lg, err := zap.NewProduction() + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } @@ -193,7 +194,7 @@ func newClientCfg(endpoints []string, dialTimeout, keepAliveTime, keepAliveTimeo // set tls if any one tls option set var cfgtls *transport.TLSInfo tlsinfo := transport.TLSInfo{} - tlsinfo.Logger, _ = zap.NewProduction() + tlsinfo.Logger, _ = logutil.CreateDefaultZapLogger(zap.InfoLevel) if scfg.cert != "" { tlsinfo.CertFile = scfg.cert cfgtls = &tlsinfo diff --git a/etcdctl/ctlv3/command/move_leader_command.go b/etcdctl/ctlv3/command/move_leader_command.go index 098c897cd7a9..b70fabdee3ca 100644 --- a/etcdctl/ctlv3/command/move_leader_command.go +++ b/etcdctl/ctlv3/command/move_leader_command.go @@ -43,7 +43,8 @@ func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } - c := mustClientFromCmd(cmd) + cfg := clientConfigFromCmd(cmd) + c := cfg.mustClient() eps := c.Endpoints() c.Close() @@ -53,7 +54,6 @@ func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) { var leaderCli *clientv3.Client var leaderID uint64 for _, ep := range eps { - cfg := clientConfigFromCmd(cmd) cfg.endpoints = []string{ep} cli := cfg.mustClient() resp, serr := cli.Status(ctx, ep) diff --git a/etcdctl/ctlv3/command/printer.go b/etcdctl/ctlv3/command/printer.go index 2d31d9ec8c6b..e9edaf0cd43b 100644 --- a/etcdctl/ctlv3/command/printer.go +++ b/etcdctl/ctlv3/command/printer.go @@ -107,6 +107,9 @@ func (p *printerRPC) MemberAdd(r v3.MemberAddResponse) { p.p((*pb.MemberAddRespo func (p *printerRPC) MemberRemove(id uint64, r v3.MemberRemoveResponse) { p.p((*pb.MemberRemoveResponse)(&r)) } +func (p *printerRPC) MemberPromote(id uint64, r v3.MemberPromoteResponse) { + p.p((*pb.MemberPromoteResponse)(&r)) +} func (p *printerRPC) MemberUpdate(id uint64, r v3.MemberUpdateResponse) { p.p((*pb.MemberUpdateResponse)(&r)) } diff --git a/etcdctl/ctlv3/command/printer_json.go b/etcdctl/ctlv3/command/printer_json.go index ca90a4a311c5..4c75c85a8c7d 100644 --- a/etcdctl/ctlv3/command/printer_json.go +++ b/etcdctl/ctlv3/command/printer_json.go @@ -67,7 +67,7 @@ func printMemberListWithHexJSON(r clientv3.MemberListResponse) { b = strconv.AppendUint(nil, r.Header.MemberId, 16) buffer.Write(b) buffer.WriteString("\",\"raft_term\":") - b = strconv.AppendUint(nil, r.Header.RaftTerm, 16) + b = strconv.AppendUint(nil, r.Header.RaftTerm, 10) buffer.Write(b) buffer.WriteByte('}') for i := 0; i < len(r.Members); i++ { diff --git a/etcdctl/ctlv3/command/snapshot_command.go b/etcdctl/ctlv3/command/snapshot_command.go index e5d3f3f1c56c..eb0227d81810 100644 --- a/etcdctl/ctlv3/command/snapshot_command.go +++ b/etcdctl/ctlv3/command/snapshot_command.go @@ -20,6 +20,7 @@ import ( "os" "github.com/spf13/cobra" + "go.etcd.io/etcd/client/pkg/v3/logutil" snapshot "go.etcd.io/etcd/client/v3/snapshot" "go.etcd.io/etcd/etcdutl/v3/etcdutl" "go.etcd.io/etcd/pkg/v3/cobrautl" @@ -39,6 +40,9 @@ var ( restorePeerURLs string restoreName string skipHashCheck bool + markCompacted bool + revisionBump uint64 + initialMmapSize uint64 ) // NewSnapshotCommand returns the cobra command for "snapshot". @@ -88,6 +92,9 @@ func NewSnapshotRestoreCommand() *cobra.Command { cmd.Flags().StringVar(&restorePeerURLs, "initial-advertise-peer-urls", defaultInitialAdvertisePeerURLs, "List of this member's peer URLs to advertise to the rest of the cluster") cmd.Flags().StringVar(&restoreName, "name", defaultName, "Human-readable name for this member") cmd.Flags().BoolVar(&skipHashCheck, "skip-hash-check", false, "Ignore snapshot integrity hash value (required if copied from data directory)") + cmd.Flags().Uint64Var(&revisionBump, "bump-revision", 0, "How much to increase the latest revision after restore") + cmd.Flags().BoolVar(&markCompacted, "mark-compacted", false, "Mark the latest revision after restore as the point of scheduled compaction (required if --bump-revision > 0, disallowed otherwise)") + cmd.Flags().Uint64Var(&initialMmapSize, "initial-memory-map-size", initialMmapSize, "Initial memory map size of the database in bytes. It uses the default value if not defined or defined to 0") return cmd } @@ -98,7 +105,7 @@ func snapshotSaveCommandFunc(cmd *cobra.Command, args []string) { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } - lg, err := zap.NewProduction() + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } @@ -126,7 +133,7 @@ func snapshotStatusCommandFunc(cmd *cobra.Command, args []string) { func snapshotRestoreCommandFunc(cmd *cobra.Command, args []string) { fmt.Fprintf(os.Stderr, "Deprecated: Use `etcdutl snapshot restore` instead.\n\n") etcdutl.SnapshotRestoreCommandFunc(restoreCluster, restoreClusterToken, restoreDataDir, restoreWalDir, - restorePeerURLs, restoreName, skipHashCheck, args) + restorePeerURLs, restoreName, skipHashCheck, initialMmapSize, revisionBump, markCompacted, args) } func initialClusterFromName(name string) string { diff --git a/etcdctl/go.mod b/etcdctl/go.mod index 04001e5cee8e..5c26cb60a228 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -1,6 +1,8 @@ module go.etcd.io/etcd/etcdctl/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 require ( github.com/bgentry/speakeasy v0.1.0 @@ -9,18 +11,63 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.4 - go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/v2 v2.305.0-alpha.0 - go.etcd.io/etcd/client/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/etcdutl/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 - go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 + go.etcd.io/etcd/api/v3 v3.5.21 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 + go.etcd.io/etcd/client/v2 v2.305.21 + go.etcd.io/etcd/client/v3 v3.5.21 + go.etcd.io/etcd/etcdutl/v3 v3.5.21 + go.etcd.io/etcd/pkg/v3 v3.5.21 + go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba - google.golang.org/grpc v1.37.0 + google.golang.org/grpc v1.59.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 ) +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.11 // indirect + go.etcd.io/etcd/raft/v3 v3.5.21 // indirect + go.etcd.io/etcd/server/v3 v3.5.21 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/protobuf v1.33.0 // indirect +) + replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 90c5668201bc..96b822516378 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -4,9 +4,13 @@ cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSR cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -19,11 +23,10 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -31,37 +34,26 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.1 h1:7OO2CXWMYNDdaAzP51t4lCCZWwpQHmvPbm9sxWjm3So= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,80 +61,74 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -168,23 +154,24 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -210,10 +197,10 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -226,28 +213,27 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -257,11 +243,10 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -277,12 +262,11 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -290,38 +274,27 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0 h1:FPuyGXkE6qPKJ71PyS0sdXuxUvYGXAXxV0XHpx0qjHE= -go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 h1:040c3dLNhgFQkoojH2AMpHCy4SrvhmxdU72d9GLGGE0= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -329,8 +302,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -344,7 +317,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -368,15 +340,15 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -384,6 +356,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -398,26 +373,27 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -426,23 +402,19 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -452,8 +424,9 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -463,33 +436,26 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -498,23 +464,22 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/etcdctl/main.go b/etcdctl/main.go index 7fbd771f6480..25b210735a14 100644 --- a/etcdctl/main.go +++ b/etcdctl/main.go @@ -27,7 +27,7 @@ const ( apiEnv = "ETCDCTL_API" ) -/** +/* mainWithError is fully analogous to main, but instead of signaling errors by os.Exit, it exposes the error explicitly, such that test-logic can intercept control to e.g. dump coverage data (even for test-for-failure scenarios). diff --git a/etcdctl/main_test.go b/etcdctl/main_test.go index 604241cacaff..3ce4b6408872 100644 --- a/etcdctl/main_test.go +++ b/etcdctl/main_test.go @@ -36,7 +36,7 @@ func SplitTestArgs(args []string) (testArgs, appArgs []string) { return } -// Empty test to avoid no-tests warning. +// TestEmpty to avoid no-tests warning. func TestEmpty(t *testing.T) {} /** diff --git a/etcdutl/ctl.go b/etcdutl/ctl.go index a044547c63c6..dbdc5cc7810d 100644 --- a/etcdutl/ctl.go +++ b/etcdutl/ctl.go @@ -41,6 +41,7 @@ func init() { etcdutl.NewDefragCommand(), etcdutl.NewSnapshotCommand(), etcdutl.NewVersionCommand(), + etcdutl.NewCheckCommand(), ) } diff --git a/etcdutl/etcdutl/backup_command.go b/etcdutl/etcdutl/backup_command.go index c09bcf14a796..0fa1879ed4f9 100644 --- a/etcdutl/etcdutl/backup_command.go +++ b/etcdutl/etcdutl/backup_command.go @@ -114,7 +114,7 @@ func HandleBackup(withV3 bool, srcDir string, destDir string, srcWAL string, des destWAL = datadir.ToWalDir(destDir) } - if err := fileutil.CreateDirAll(destSnap); err != nil { + if err := fileutil.CreateDirAll(lg, destSnap); err != nil { lg.Fatal("failed creating backup snapshot dir", zap.String("dest-snap", destSnap), zap.Error(err)) } @@ -319,10 +319,10 @@ func saveDB(lg *zap.Logger, destDB, srcDB string, idx uint64, term uint64, desir if !v3 { tx := be.BatchTx() - tx.Lock() + tx.LockOutsideApply() defer tx.Unlock() cindex.UnsafeCreateMetaBucket(tx) - cindex.UnsafeUpdateConsistentIndex(tx, idx, term, false) + cindex.UnsafeUpdateConsistentIndex(tx, idx, term) } else { // Thanks to translateWAL not moving entries, but just replacing them with // 'empty', there is no need to update the consistency index. diff --git a/etcdutl/etcdutl/check_command.go b/etcdutl/etcdutl/check_command.go new file mode 100644 index 000000000000..2ea6fb29ae24 --- /dev/null +++ b/etcdutl/etcdutl/check_command.go @@ -0,0 +1,111 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdutl + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + "go.etcd.io/etcd/pkg/v3/cobrautl" + "go.etcd.io/etcd/server/v3/datadir" + "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" + "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" + "go.etcd.io/etcd/server/v3/wal" +) + +// NewCheckCommand returns the cobra command for "check". +func NewCheckCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "check ", + Short: "commands for checking properties", + } + cmd.AddCommand(NewCheckV2StoreCommand()) + return cmd +} + +var ( + argCheckV2StoreDataDir string +) + +// NewCheckV2StoreCommand returns the cobra command for "check v2store". +func NewCheckV2StoreCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "v2store", + Short: "Check custom content in v2store", + Run: checkV2StoreRunFunc, + } + cmd.Flags().StringVar(&argCheckV2StoreDataDir, "data-dir", "", "Required. A data directory not in use by etcd.") + cmd.MarkFlagRequired("data-dir") + return cmd +} + +func checkV2StoreRunFunc(_ *cobra.Command, _ []string) { + err := checkV2StoreDataDir(argCheckV2StoreDataDir) + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + fmt.Println("No custom content found in v2store.") +} + +func checkV2StoreDataDir(dataDir string) error { + var ( + lg = GetLogger() + + walDir = datadir.ToWalDir(dataDir) + snapDir = datadir.ToSnapDir(dataDir) + ) + + walSnaps, err := wal.ValidSnapshotEntries(lg, walDir) + if err != nil { + if errors.Is(err, wal.ErrFileNotFound) { + return nil + } + return err + } + + ss := snap.New(lg, snapDir) + snapshot, err := ss.LoadNewestAvailable(walSnaps) + if err != nil { + if errors.Is(err, snap.ErrNoSnapshot) { + return nil + } + return err + } + if snapshot == nil { + return nil + } + + st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) + + if err := st.Recovery(snapshot.Data); err != nil { + return fmt.Errorf("failed to recover v2store from snapshot: %w", err) + } + return assertNoV2StoreContent(st) +} + +func assertNoV2StoreContent(st v2store.Store) error { + metaOnly, err := membership.IsMetaStoreOnly(st) + if err != nil { + return err + } + if metaOnly { + return nil + } + return fmt.Errorf("detected custom content in v2store") +} diff --git a/etcdutl/etcdutl/common.go b/etcdutl/etcdutl/common.go index 4b4a198aaf82..281058bc24e9 100644 --- a/etcdutl/etcdutl/common.go +++ b/etcdutl/etcdutl/common.go @@ -15,13 +15,14 @@ package etcdutl import ( + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) func GetLogger() *zap.Logger { - config := zap.NewProductionConfig() + config := logutil.DefaultZapLoggerConfig config.Encoding = "console" config.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder lg, err := config.Build() diff --git a/etcdutl/etcdutl/snapshot_command.go b/etcdutl/etcdutl/snapshot_command.go index 94ab2a5ac927..a7d9397bccc1 100644 --- a/etcdutl/etcdutl/snapshot_command.go +++ b/etcdutl/etcdutl/snapshot_command.go @@ -21,6 +21,7 @@ import ( "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/datadir" + "go.etcd.io/etcd/server/v3/mvcc/backend" "github.com/spf13/cobra" ) @@ -38,6 +39,9 @@ var ( restorePeerURLs string restoreName string skipHashCheck bool + initialMmapSize = backend.InitialMmapSize + markCompacted bool + revisionBump uint64 ) // NewSnapshotCommand returns the cobra command for "snapshot". @@ -91,6 +95,9 @@ func NewSnapshotRestoreCommand() *cobra.Command { cmd.Flags().StringVar(&restorePeerURLs, "initial-advertise-peer-urls", defaultInitialAdvertisePeerURLs, "List of this member's peer URLs to advertise to the rest of the cluster") cmd.Flags().StringVar(&restoreName, "name", defaultName, "Human-readable name for this member") cmd.Flags().BoolVar(&skipHashCheck, "skip-hash-check", false, "Ignore snapshot integrity hash value (required if copied from data directory)") + cmd.Flags().Uint64Var(&initialMmapSize, "initial-memory-map-size", initialMmapSize, "Initial memory map size of the database in bytes. It uses the default value if not defined or defined to 0") + cmd.Flags().Uint64Var(&revisionBump, "bump-revision", 0, "How much to increase the latest revision after restore") + cmd.Flags().BoolVar(&markCompacted, "mark-compacted", false, "Mark the latest revision after restore as the point of scheduled compaction (required if --bump-revision > 0, disallowed otherwise)") cmd.MarkFlagRequired("data-dir") @@ -115,7 +122,7 @@ func SnapshotStatusCommandFunc(cmd *cobra.Command, args []string) { func snapshotRestoreCommandFunc(_ *cobra.Command, args []string) { SnapshotRestoreCommandFunc(restoreCluster, restoreClusterToken, restoreDataDir, restoreWalDir, - restorePeerURLs, restoreName, skipHashCheck, args) + restorePeerURLs, restoreName, skipHashCheck, initialMmapSize, revisionBump, markCompacted, args) } func SnapshotRestoreCommandFunc(restoreCluster string, @@ -125,12 +132,20 @@ func SnapshotRestoreCommandFunc(restoreCluster string, restorePeerURLs string, restoreName string, skipHashCheck bool, + initialMmapSize uint64, + revisionBump uint64, + markCompacted bool, args []string) { if len(args) != 1 { err := fmt.Errorf("snapshot restore requires exactly one argument") cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } + if (revisionBump == 0 && markCompacted) || (revisionBump > 0 && !markCompacted) { + err := fmt.Errorf("--mark-compacted required if --revision-bump > 0") + cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) + } + dataDir := restoreDataDir if dataDir == "" { dataDir = restoreName + ".etcd" @@ -153,6 +168,9 @@ func SnapshotRestoreCommandFunc(restoreCluster string, InitialCluster: restoreCluster, InitialClusterToken: restoreClusterToken, SkipHashCheck: skipHashCheck, + InitialMmapSize: initialMmapSize, + RevisionBump: revisionBump, + MarkCompacted: markCompacted, }); err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } diff --git a/etcdutl/go.mod b/etcdutl/go.mod index 6e61a0e4ca75..d24ca05eac79 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -1,6 +1,8 @@ module go.etcd.io/etcd/etcdutl/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 replace ( go.etcd.io/etcd/api/v3 => ../api @@ -24,12 +26,55 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.1.3 - go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0 - go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/server/v3 v3.5.0-alpha.0 - go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 + go.etcd.io/bbolt v1.3.11 + go.etcd.io/etcd/api/v3 v3.5.21 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 + go.etcd.io/etcd/client/v3 v3.5.21 + go.etcd.io/etcd/pkg/v3 v3.5.21 + go.etcd.io/etcd/raft/v3 v3.5.21 + go.etcd.io/etcd/server/v3 v3.5.21 + go.uber.org/zap v1.17.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/etcd/client/v2 v2.305.21 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.59.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/etcdutl/go.sum b/etcdutl/go.sum index d44b8a0a84d1..a076e08a1140 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -4,9 +4,13 @@ cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSR cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -19,46 +23,34 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.1 h1:7OO2CXWMYNDdaAzP51t4lCCZWwpQHmvPbm9sxWjm3So= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,79 +58,73 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -164,23 +150,24 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -204,10 +191,10 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -220,28 +207,27 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -249,11 +235,10 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -269,49 +254,37 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0 h1:FPuyGXkE6qPKJ71PyS0sdXuxUvYGXAXxV0XHpx0qjHE= -go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 h1:040c3dLNhgFQkoojH2AMpHCy4SrvhmxdU72d9GLGGE0= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -319,8 +292,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -334,7 +307,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -358,15 +330,15 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -374,6 +346,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -388,26 +363,27 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -416,23 +392,19 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -442,8 +414,9 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -453,56 +426,48 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/etcdutl/main_test.go b/etcdutl/main_test.go index b54b2ba234fb..05d67e7138ac 100644 --- a/etcdutl/main_test.go +++ b/etcdutl/main_test.go @@ -36,7 +36,7 @@ func SplitTestArgs(args []string) (testArgs, appArgs []string) { return } -// Empty test to avoid no-tests warning. +// TestEmpty to avoid no-tests warning. func TestEmpty(t *testing.T) {} /** diff --git a/etcdutl/snapshot/util.go b/etcdutl/snapshot/util.go index 2c1fae21fa15..a4f3569c6884 100644 --- a/etcdutl/snapshot/util.go +++ b/etcdutl/snapshot/util.go @@ -23,9 +23,31 @@ type revision struct { sub int64 } +// GreaterThan should be synced with function in server +// https://github.com/etcd-io/etcd/blob/main/server/storage/mvcc/revision.go +func (a revision) GreaterThan(b revision) bool { + if a.main > b.main { + return true + } + if a.main < b.main { + return false + } + return a.sub > b.sub +} + +// bytesToRev should be synced with function in server +// https://github.com/etcd-io/etcd/blob/main/server/storage/mvcc/revision.go func bytesToRev(bytes []byte) revision { return revision{ main: int64(binary.BigEndian.Uint64(bytes[0:8])), sub: int64(binary.BigEndian.Uint64(bytes[9:])), } } + +// revToBytes should be synced with function in server +// https://github.com/etcd-io/etcd/blob/main/server/storage/mvcc/revision.go +func revToBytes(bytes []byte, rev revision) { + binary.BigEndian.PutUint64(bytes[0:8], uint64(rev.main)) + bytes[8] = '_' + binary.BigEndian.PutUint64(bytes[9:], uint64(rev.sub)) +} diff --git a/etcdutl/snapshot/v3_snapshot.go b/etcdutl/snapshot/v3_snapshot.go index 9272a8f0b13e..882a29dc49c3 100644 --- a/etcdutl/snapshot/v3_snapshot.go +++ b/etcdutl/snapshot/v3_snapshot.go @@ -30,7 +30,7 @@ import ( "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/snapshot" "go.etcd.io/etcd/raft/v3" "go.etcd.io/etcd/raft/v3/raftpb" @@ -40,7 +40,9 @@ import ( "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/etcdserver/cindex" + "go.etcd.io/etcd/server/v3/mvcc" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/mvcc/buckets" "go.etcd.io/etcd/server/v3/verify" "go.etcd.io/etcd/server/v3/wal" "go.etcd.io/etcd/server/v3/wal/walpb" @@ -69,9 +71,6 @@ type Manager interface { // NewV3 returns a new snapshot Manager for v3.x snapshot. func NewV3(lg *zap.Logger) Manager { - if lg == nil { - lg = zap.NewExample() - } return &v3Manager{lg: lg} } @@ -84,7 +83,8 @@ type v3Manager struct { snapDir string cl *membership.RaftCluster - skipHashCheck bool + skipHashCheck bool + initialMmapSize uint64 } // hasChecksum returns "true" if the file size "n" @@ -197,6 +197,19 @@ type RestoreConfig struct { // SkipHashCheck is "true" to ignore snapshot integrity hash value // (required if copied from data directory). SkipHashCheck bool + + // InitialMmapSize is the database initial memory map size. + InitialMmapSize uint64 + + // RevisionBump is the amount to increase the latest revision after restore, + // to allow administrators to trick clients into thinking that revision never decreased. + // If 0, revision bumping is skipped. + // (required if MarkCompacted == true) + RevisionBump uint64 + + // MarkCompacted is "true" to mark the latest revision as compacted. + // (required if RevisionBump > 0) + MarkCompacted bool } // Restore restores a new etcd data directory from given snapshot file. @@ -247,6 +260,7 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error { s.walDir = walDir s.snapDir = filepath.Join(dataDir, "member", "snap") s.skipHashCheck = cfg.SkipHashCheck + s.initialMmapSize = cfg.InitialMmapSize s.lg.Info( "restoring snapshot", @@ -254,12 +268,19 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error { zap.String("wal-dir", s.walDir), zap.String("data-dir", dataDir), zap.String("snap-dir", s.snapDir), - zap.Stack("stack"), + zap.Uint64("initial-memory-map-size", s.initialMmapSize), ) if err = s.saveDB(); err != nil { return err } + + if cfg.MarkCompacted && cfg.RevisionBump > 0 { + if err = s.modifyLatestRevision(cfg.RevisionBump); err != nil { + return err + } + } + hardstate, err := s.saveWALAndSnap() if err != nil { return err @@ -275,6 +296,7 @@ func (s *v3Manager) Restore(cfg RestoreConfig) error { zap.String("wal-dir", s.walDir), zap.String("data-dir", dataDir), zap.String("snap-dir", s.snapDir), + zap.Uint64("initial-memory-map-size", s.initialMmapSize), ) return verify.VerifyIfEnabled(verify.Config{ @@ -295,7 +317,7 @@ func (s *v3Manager) saveDB() error { return err } - be := backend.NewDefaultBackend(s.outDbPath()) + be := backend.NewDefaultBackend(s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() err = membership.TrimMembershipFromBackend(s.lg, be) @@ -306,6 +328,70 @@ func (s *v3Manager) saveDB() error { return nil } +// modifyLatestRevision can increase the latest revision by the given amount and sets the scheduled compaction +// to that revision so that the server will consider this revision compacted. +func (s *v3Manager) modifyLatestRevision(bumpAmount uint64) error { + be := backend.NewDefaultBackend(s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) + defer func() { + be.ForceCommit() + be.Close() + }() + + tx := be.BatchTx() + tx.LockOutsideApply() + defer tx.Unlock() + + latest, err := s.unsafeGetLatestRevision(tx) + if err != nil { + return err + } + + latest = s.unsafeBumpRevision(tx, latest, int64(bumpAmount)) + s.unsafeMarkRevisionCompacted(tx, latest) + + return nil +} + +func (s *v3Manager) unsafeBumpRevision(tx backend.BatchTx, latest revision, amount int64) revision { + s.lg.Info( + "bumping latest revision", + zap.Int64("latest-revision", latest.main), + zap.Int64("bump-amount", amount), + zap.Int64("new-latest-revision", latest.main+amount), + ) + + latest.main += amount + latest.sub = 0 + k := make([]byte, 17) + revToBytes(k, latest) + tx.UnsafePut(buckets.Key, k, []byte{}) + + return latest +} + +func (s *v3Manager) unsafeMarkRevisionCompacted(tx backend.BatchTx, latest revision) { + s.lg.Info( + "marking revision compacted", + zap.Int64("revision", latest.main), + ) + + mvcc.UnsafeSetScheduledCompact(tx, latest.main) +} + +func (s *v3Manager) unsafeGetLatestRevision(tx backend.BatchTx) (revision, error) { + var latest revision + err := tx.UnsafeForEach(buckets.Key, func(k, _ []byte) (err error) { + rev := bytesToRev(k) + + if rev.GreaterThan(latest) { + latest = rev + } + + return nil + }) + return latest, err +} + func (s *v3Manager) copyAndVerifyDB() error { srcf, ferr := os.Open(s.srcDbPath) if ferr != nil { @@ -325,7 +411,7 @@ func (s *v3Manager) copyAndVerifyDB() error { return err } - if err := fileutil.CreateDirAll(s.snapDir); err != nil { + if err := fileutil.CreateDirAll(s.lg, s.snapDir); err != nil { return err } @@ -335,13 +421,8 @@ func (s *v3Manager) copyAndVerifyDB() error { if dberr != nil { return dberr } - dbClosed := false - defer func() { - if !dbClosed { - db.Close() - dbClosed = true - } - }() + defer db.Close() + if _, err := io.Copy(db, srcf); err != nil { return err } @@ -378,7 +459,7 @@ func (s *v3Manager) copyAndVerifyDB() error { } // db hash is OK, can now modify DB so it can be part of a new cluster - db.Close() + return nil } @@ -386,14 +467,14 @@ func (s *v3Manager) copyAndVerifyDB() error { // // TODO: This code ignores learners !!! func (s *v3Manager) saveWALAndSnap() (*raftpb.HardState, error) { - if err := fileutil.CreateDirAll(s.walDir); err != nil { + if err := fileutil.CreateDirAll(s.lg, s.walDir); err != nil { return nil, err } // add members again to persist them to the store we create. st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) s.cl.SetStore(st) - be := backend.NewDefaultBackend(s.outDbPath()) + be := backend.NewDefaultBackend(s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() s.cl.SetBackend(be) for _, m := range s.cl.Members() { @@ -479,6 +560,6 @@ func (s *v3Manager) updateCIndex(commit uint64, term uint64) error { be := backend.NewDefaultBackend(s.outDbPath()) defer be.Close() - cindex.UpdateConsistentIndex(be.BatchTx(), commit, term, false) + cindex.UpdateConsistentIndex(be.BatchTx(), commit, term) return nil } diff --git a/go.mod b/go.mod index 4e847849da8a..b5759ef746f2 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,11 @@ module go.etcd.io/etcd/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 replace ( + github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 go.etcd.io/etcd/api/v3 => ./api go.etcd.io/etcd/client/pkg/v3 => ./client/pkg go.etcd.io/etcd/client/v2 => ./client/v2 @@ -18,20 +21,93 @@ replace ( require ( github.com/bgentry/speakeasy v0.1.0 github.com/dustin/go-humanize v1.0.0 - github.com/spf13/cobra v1.1.3 - go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0 - go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/client/v2 v2.305.0-alpha.0 - go.etcd.io/etcd/client/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/etcdctl/v3 v3.0.0-00010101000000-000000000000 - go.etcd.io/etcd/etcdutl/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/server/v3 v3.5.0-alpha.0 - go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 - go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 + github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835 + github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 + go.etcd.io/bbolt v1.3.11 + go.etcd.io/etcd/api/v3 v3.5.21 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 + go.etcd.io/etcd/client/v2 v2.305.21 + go.etcd.io/etcd/client/v3 v3.5.21 + go.etcd.io/etcd/etcdctl/v3 v3.5.21 + go.etcd.io/etcd/etcdutl/v3 v3.5.21 + go.etcd.io/etcd/pkg/v3 v3.5.21 + go.etcd.io/etcd/raft/v3 v3.5.21 + go.etcd.io/etcd/server/v3 v3.5.21 + go.etcd.io/etcd/tests/v3 v3.5.21 + go.uber.org/zap v1.17.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba - google.golang.org/grpc v1.37.0 + google.golang.org/grpc v1.64.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 ) + +require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/google/cel-go v0.17.8 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/onsi/ginkgo/v2 v2.25.1 // indirect + github.com/onsi/gomega v1.38.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/soheilhy/cmux v0.1.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/sdk v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools v0.36.0 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 // indirect + google.golang.org/protobuf v1.36.7 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/go.sum b/go.sum index d35fb315f0cd..4c4e18b58416 100644 --- a/go.sum +++ b/go.sum @@ -1,109 +1,84 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.1 h1:7OO2CXWMYNDdaAzP51t4lCCZWwpQHmvPbm9sxWjm3So= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -113,104 +88,68 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -218,320 +157,243 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835 h1:rkqIIfdYYkasXbF2XKVgh/3f1mhjSQK9By8WtVMgYo8= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20250916161632-d81c09058835/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0 h1:FPuyGXkE6qPKJ71PyS0sdXuxUvYGXAXxV0XHpx0qjHE= -go.etcd.io/bbolt v1.3.6-0.20210426205525-9c92be978ae0/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= +go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.20.0 h1:5Jf6imeFZlZtKv9Qbo6qt2ZkmWtdWx/wzcCbNUlAWGM= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 h1:040c3dLNhgFQkoojH2AMpHCy4SrvhmxdU72d9GLGGE0= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61 h1:N9BgCIAUvn/M+p4NJccWPWb3BWh88+zyL0ll9HgbEeM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240924160255-9d4c2d233b61/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/openshift-hack/rebase.sh b/openshift-hack/rebase.sh new file mode 100755 index 000000000000..7e79e2c72dd3 --- /dev/null +++ b/openshift-hack/rebase.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# READ FIRST BEFORE USING THIS SCRIPT +# +# This script requires jq, git, podman and bash to work properly (dependencies are checked for you). +# The Github CLI "gh" is optional, but convenient to create a pull request automatically at the end. +# +# The usage is described in /REBASE.openshift.md. + +# validate input args --etcd-tag=v3.5.4 --openshift-release=openshift-4.12 --jira-id=666 +etcd_tag="" +openshift_release="" +jira_id="" + +usage() { + echo "Available arguments:" + echo " --etcd-tag (required) Example: --etcd-tag=v3.4.20" + echo " --openshift-release (required) Example: --openshift-release=openshift-4.12" + echo " --jira-id (optional) creates new PR against openshift/etcd:${openshift-release}: Example: --jira-id=666" +} + +for i in "$@"; do + case $i in + --etcd-tag=*) + etcd_tag="${i#*=}" + shift + ;; + --openshift-release=*) + openshift_release="${i#*=}" + shift + ;; + --jira-id=*) + jira_id="${i#*=}" + shift + ;; + *) + usage + exit 1 + ;; + esac +done + +if [ -z "${etcd_tag}" ]; then + echo "Required argument missing: --etcd-tag" + echo "" + usage + exit 1 +fi + +if [ -z "${openshift_release}" ]; then + echo "Required argument missing: --openshift-release" + echo "" + usage + exit 1 +fi + +echo "Processed arguments are:" +echo "--etcd_tag=${etcd_tag}" +echo "--openshift_release=${openshift_release}" +echo "--jira_id=${jira_id}" + +# prerequisites (check git, podman, ... is present) +if ! command -v git &>/dev/null; then + echo "git not installed, exiting" + exit 1 +fi + +if ! command -v jq &>/dev/null; then + echo "jq not installed, exiting" + exit 1 +fi + +if ! command -v podman &>/dev/null; then + echo "podman not installed, exiting" + exit 1 +fi + +# make sure we're in "etcd" dir, but we also allow openshift-etcd +if [[ $(basename "$PWD") != "etcd" && $(basename "$PWD") != "openshift-etcd" ]]; then + echo "Not in etcd dir, exiting" + exit 1 +fi + +origin=$(git remote get-url origin) +if [[ "$origin" =~ .*etcd-io/etcd.* || "$origin" =~ .*openshift/etcd.* ]]; then + echo "cannot rebase against etcd-io/etcd or openshift/etcd! found: ${origin}, exiting" + exit 1 +fi + +# fetch remote https://github.com/etcd-io/etcd +git remote add upstream git@github.com:etcd-io/etcd.git +git fetch upstream --tags -f +# fetch remote https://github.com/openshift/etcd +git remote add openshift git@github.com:openshift/etcd.git +git fetch openshift + +# clean checkout of the remote openshift release +git branch -D "openshift/$openshift_release" +git checkout --track "openshift/$openshift_release" +git pull openshift "$openshift_release" + +# that should give us the latest (or highest version) etcd tag +# This is a bit experimental for the future, but works across all the current release branches +etcd_forkpoint=$(git tag --merged | sort -V | tail -2 | head -1) +if [[ "$etcd_forkpoint" == "$etcd_tag" ]]; then + echo "forkpoint $etcd_forkpoint matches given etcd tag, no rebase necessary" + exit 1 +fi + +echo "running: \`git rebase --rebase-merges --fork-point $etcd_forkpoint $etcd_tag\`" +git rebase --rebase-merges --fork-point "$etcd_forkpoint" "$etcd_tag" +echo "running: \`git merge $openshift_release\`" +git merge "$openshift_release" + +# shellcheck disable=SC2181 +if [ $? -eq 0 ]; then + echo "No conflicts detected. Automatic merge looks to have succeeded" +else + # commit conflicts + git commit -a + # resolve conflicts + git status + # TODO(tjungblu): we follow-up with a more automated approach: + # - 2/3s of conflicts stem from go.mod/sum, which can be resolved deterministically + # - the large majority of the remainder are vendor/generation conflicts + # - only very few cases require manual intervention due to conflicting business logic + echo "Resolve conflicts manually in another terminal, only then continue" + + # wait for user interaction + read -n 1 -s -r -p "PRESS ANY KEY TO CONTINUE" + + # TODO(tjungblu): verify that the conflicts have been resolved + git commit -am "UPSTREAM: : manually resolve conflicts" +fi + +# ensure we always use the correct openshift release + golang combination +go_mod_go_ver=$(grep -E 'go 1\.[1-9][0-9]?' go.mod | sed -E 's/go (1\.[1-9][0-9]?)/\1/') +tag="rhel-8-release-golang-${go_mod_go_ver}-openshift-${openshift_release#release-}" +echo "> go mod tidy" +podman run -it --rm -v "$(pwd):/go/etcd:Z" \ + --workdir=/go/etcd \ + "registry.ci.openshift.org/openshift/release:$tag" \ + go mod tidy + +# shellcheck disable=SC2181 +if [ $? -ne 0 ]; then + echo "go mod tidy failed, is any dependency missing?" + exit 1 +fi + +git add -A +git commit -m "UPSTREAM: : go mod tidy" + +remote_branch="rebase-$etcd_tag" +git push origin "$openshift_release:$remote_branch" + +XY=$(echo "$etcd_tag" | sed -E "s/v(1\.[0-9]+)\.[0-9]+/\1/") +ver=$(echo "$etcd_tag" | sed "s/\.//g") +link="https://github.com/etcd-io/etcd/blob/master/CHANGELOG/CHANGELOG-$XY.md#$ver" +if [ -n "${jira_id}" ]; then + if command -v gh &>/dev/null; then + XY=$(echo "$etcd_tag" | sed -E "s/v(1\.[0-9]+)\.[0-9]+/\1/") + ver=$(echo "$etcd_tag" | sed "s/\.//g") + link="https://github.com/etcd-io/etcd/blob/master/CHANGELOG/CHANGELOG-$XY.md#$ver" + + # opens a web browser, because we can't properly create PRs against remote repositories with the GH CLI (yet): + # https://github.com/cli/cli/issues/2691 + gh pr create \ + --title "OCPBUGS-$jira_id: Rebase $etcd_tag" \ + --body "CHANGELOG $link" \ + --web + + fi +fi diff --git a/openshift-tools/discover-etcd-initial-cluster/main.go b/openshift-tools/discover-etcd-initial-cluster/main.go new file mode 100644 index 000000000000..544c666d91d6 --- /dev/null +++ b/openshift-tools/discover-etcd-initial-cluster/main.go @@ -0,0 +1,35 @@ +package main + +import ( + goflag "flag" + "fmt" + "math/rand" + "os" + "strings" + "time" + + "github.com/spf13/pflag" + discover_etcd_initial_cluster "go.etcd.io/etcd/v3/openshift-tools/pkg/discover-etcd-initial-cluster" +) + +// copy from `utilflag "k8s.io/component-base/cli/flag"` +// WordSepNormalizeFunc changes all flags that contain "_" separators +func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { + if strings.Contains(name, "_") { + return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) + } + return pflag.NormalizedName(name) +} + +func main() { + rand.Seed(time.Now().UTC().UnixNano()) + + pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + + command := discover_etcd_initial_cluster.NewDiscoverEtcdInitialClusterCommand() + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} diff --git a/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go new file mode 100644 index 000000000000..0c6bf6163e9a --- /dev/null +++ b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster.go @@ -0,0 +1,393 @@ +package discover_etcd_initial_cluster + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "go.uber.org/zap" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "google.golang.org/grpc" + + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/transport" + "go.etcd.io/etcd/client/v3" +) + +type DiscoverEtcdInitialClusterOptions struct { + // TargetPeerURLHost is the host portion of the peer URL. It is used to match on. (either IP or hostname) + TargetPeerURLHost string + // TargetPeerURLScheme is the host scheme of the peer URL. + TargetPeerURLScheme string + // TargetPeerURLPort is the host port of the peer URL. + TargetPeerURLPort string + // TargetName is the name to assign to this peer if we create it. + TargetName string + + // CABundleFile is the file to use to trust the etcd server + CABundleFile string + // ClientCertFile is the client cert to use to authenticate this binary to etcd + ClientCertFile string + // ClientKeyFile is the client key to use to authenticate this binary to etcd + ClientKeyFile string + // Endpoints is a list of all the endpoints to use to try to contact etcd + Endpoints []string + + // DataDir is the directory created when etcd starts the first time + DataDir string +} + +type revisionStruct struct { + ClusterId uint64 `json:"clusterId,omitempty"` +} + +func NewDiscoverEtcdInitialCluster() *DiscoverEtcdInitialClusterOptions { + return &DiscoverEtcdInitialClusterOptions{ + TargetPeerURLScheme: "https", + TargetPeerURLPort: "2380", + } +} + +func NewDiscoverEtcdInitialClusterCommand() *cobra.Command { + o := NewDiscoverEtcdInitialCluster() + + cmd := &cobra.Command{ + Use: "discover-etcd-initial-cluster", + Short: "output the value for ETCD_INITIAL_CLUSTER in openshift etcd static pod", + Long: `output the value for ETCD_INITIAL_CLUSTER in openshift etcd static pod + +Please see docs for more details: +https://github.com/openshift/cluster-etcd-operator/tree/master/docs/discover-etcd-initial-cluster.md +`, + Run: func(cmd *cobra.Command, args []string) { + if err := o.Validate(); err != nil { + fmt.Fprint(os.Stderr, err) + os.Exit(1) + } + + if err := o.Run(); err != nil { + fmt.Fprint(os.Stderr, err) + os.Exit(1) + } + }, + } + o.BindFlags(cmd.Flags()) + + return cmd +} + +func (o *DiscoverEtcdInitialClusterOptions) BindFlags(flags *pflag.FlagSet) { + flags.StringVar(&o.CABundleFile, "cacert", o.CABundleFile, "file to use to verify the identity of the etcd server") + flags.StringVar(&o.ClientCertFile, "cert", o.ClientCertFile, "client cert to use to authenticate this binary to etcd") + flags.StringVar(&o.ClientKeyFile, "key", o.ClientKeyFile, "client key to use to authenticate this binary to etcd") + flags.StringSliceVar(&o.Endpoints, "endpoints", o.Endpoints, "list of all the endpoints to use to try to contact etcd") + flags.StringVar(&o.DataDir, "data-dir", o.DataDir, "dir to stat for existence of the member directory") + flags.StringVar(&o.TargetPeerURLHost, "target-peer-url-host", o.TargetPeerURLHost, "host portion of the peer URL. It is used to match on. (either IP or hostname)") + flags.StringVar(&o.TargetName, "target-name", o.TargetName, "name to assign to this peer if we create it") +} + +func (o *DiscoverEtcdInitialClusterOptions) Validate() error { + if len(o.CABundleFile) == 0 { + return fmt.Errorf("missing --cacert") + } + if len(o.ClientCertFile) == 0 { + return fmt.Errorf("missing --cert") + } + if len(o.ClientKeyFile) == 0 { + return fmt.Errorf("missing --key") + } + if len(o.Endpoints) == 0 { + return fmt.Errorf("missing --endpoints") + } + if len(o.DataDir) == 0 { + return fmt.Errorf("missing --data-dir") + } + if len(o.TargetPeerURLHost) == 0 { + return fmt.Errorf("missing --target-peer-url-host") + } + if len(o.TargetName) == 0 { + return fmt.Errorf("missing --target-name") + } + if len(o.TargetPeerURLPort) == 0 { + return fmt.Errorf("missing TargetPeerURLPort") + } + if len(o.TargetPeerURLScheme) == 0 { + return fmt.Errorf("missing TargetPeerURLScheme") + } + return nil +} + +func (o *DiscoverEtcdInitialClusterOptions) Run() error { + var dataDirExists bool + // check if dataDir structure exists + _, err := os.Stat(filepath.Join(o.DataDir, "member/snap")) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + fmt.Fprintf(os.Stderr, "dataDir is present on %s\n", o.TargetName) + dataDirExists = true + } + + client, err := o.getClient() + + // Condition: create client fail with dataDir + // Possible reasons for this condition. + // 1.) single node etcd cluster + // 2.) transient networking problem + // 3.) on and off flow + // Result: start etcd with empty initial config + if err != nil && dataDirExists { + fmt.Fprintf(os.Stderr, "failed to create etcd client, but the server is already initialized as member %q before, starting as etcd member: %v", o.TargetName, err.Error()) + return nil + } + // Condition: create client fail, no dataDir + // Possible reasons for the condition include transient network partition. + // Result: return error and restart container + if err != nil { + return fmt.Errorf("failed to create etcd client: %v", err) + } + defer client.Close() + + localClusterIdentifier, err := o.findLocalClusterIdentifier() + if err != nil { + return fmt.Errorf("could not find local cluster id: %w", err) + } + + // the startupProbe waits for 180s, so we are giving 135s to this process and 45s to the etcd that runs after us. + for i := 0; i < 135; i++ { + fmt.Fprintf(os.Stderr, "#### attempt %d\n", i) + + // Check member list on each iteration for changes. + cluster, err := client.Cluster.(clientv3.NonLinearizeableMemberLister).NonLinearizeableMemberList(context.TODO()) + if err != nil { + fmt.Fprintf(os.Stderr, "member list request failed: %v", err) + continue + } + fmt.Fprintf(os.Stderr, "Live Cluster ID: [%s], local: [%s] \n", + strconv.FormatUint(cluster.Header.ClusterId, 16), + strconv.FormatUint(localClusterIdentifier, 16)) + logCurrentMembership(cluster.Members) + + mismatchingClusterId := localClusterIdentifier != 0 && cluster.Header.ClusterId != localClusterIdentifier + + initialCluster, memberFound, err := o.getInitialCluster( + cluster.Members, + dataDirExists, + mismatchingClusterId) + if err != nil && memberFound { + return err + } + // If member is not yet part of the cluster print to stderr and retry. + if err != nil && !memberFound { + fmt.Fprintf(os.Stderr, " %s\n#### sleeping...\n", err.Error()) + time.Sleep(1 * time.Second) + continue + } + // Empty string value for initialCluster is valid. + // this is important to go as the only resulting string to stdout, as it is creating an env var in the pod yaml + fmt.Println(initialCluster) + + return nil + } + return fmt.Errorf("timed out") +} + +func (o *DiscoverEtcdInitialClusterOptions) getInitialCluster(members []*etcdserverpb.Member, dataDirExists, mismatchingClusterId bool) (string, bool, error) { + target := url.URL{ + Scheme: o.TargetPeerURLScheme, + Host: fmt.Sprintf("%s:%s", o.TargetPeerURLHost, o.TargetPeerURLPort), + } + + targetMember, memberFound := checkTargetMember(target, members) + + // Condition: unstarted member found, no dataDir + // This member is part of the cluster but has not yet started. We know this because the name is populated at + // runtime which this member does not have. + // Result: populate initial cluster so etcd can communicate with peers during startup + if memberFound && targetMember.Name == "" && !dataDirExists { + return formatInitialCluster(o.TargetName, targetMember, members), memberFound, nil + } + + // Condition: unstarted member found with dataDir + // This member is part of the cluster but has not yet started, yet has a dataDir. + // Result: archive old dataDir and return error which will restart container + if memberFound && targetMember.Name == "" && dataDirExists { + archivedDir, err := archiveDataDir(o.DataDir) + if err != nil { + return "", memberFound, err + } + return "", memberFound, fmt.Errorf("member %q is unstarted but previous members dataDir exists: archiving to %q", target.String(), archivedDir) + } + + // Condition: started member found with dataDir + // Result: start etcd with empty initial config + if memberFound && dataDirExists { + return "", memberFound, nil + } + + // Condition: started member found, no dataDir + // A member is not actually gone forever unless it is removed from cluster with MemberRemove or the dataDir is destroyed. Since + // this is the latter. Do not let etcd start and report the condition as an error. + // Result: return error and restart container + if memberFound && !dataDirExists { + return "", memberFound, fmt.Errorf("member %q dataDir has been destroyed and must be removed from the cluster", target.String()) + } + + // Condition: member not found with dataDir + // The member has been removed from the cluster, likely from a restore operation. + // Result: if the cluster ID does not match anymore, we're archiving the data dir and returning a signal to start etcd. + // If it matches, there is a scaling problem and the datadir must be removed manually. + if !memberFound && dataDirExists { + if mismatchingClusterId { + _, err := archiveDataDir(o.DataDir) + if err != nil { + return "", memberFound, err + } + + return "", true, nil + } + + return "", memberFound, fmt.Errorf("member %q not found in member list but dataDir exists, check operator logs for possible scaling problems\n", target.String()) + } + + // Condition: member not found, no dataDir + // The member list does not reflect the target member as it is waiting to be scaled up. + // Result: retry + if !memberFound && !dataDirExists { + return "", memberFound, fmt.Errorf("member %q not found in member list, check operator logs for possible scaling problems", target.String()) + } + + return "", memberFound, nil +} + +func (o *DiscoverEtcdInitialClusterOptions) findLocalClusterIdentifier() (uint64, error) { + // we favor the revision.json as its fastest to parse, if we error here we can still rely on the WAL parsing (slow) + fromRevisionFile, revErr := o.findLocalClusterIdFromRevFile() + // zero can also be returned when the revision file does not contain the ClusterId attribute + if revErr != nil || fromRevisionFile == 0 { + fmt.Fprintf(os.Stderr, "could not parse revision.json, falling back to WAL parsing. Err=%v", revErr) + fromWal, walErr := o.findLocalClusterIdFromWal() + if walErr != nil { + return 0, fmt.Errorf("couldn't find cluster id in WAL or revision: %v", errors.Join(walErr, revErr)) + } + return fromWal, nil + } + + return fromRevisionFile, nil +} + +func (o *DiscoverEtcdInitialClusterOptions) findLocalClusterIdFromRevFile() (uint64, error) { + content, err := os.ReadFile(path.Join(o.DataDir, "revision.json")) + if err != nil { + return 0, err + } + + result := revisionStruct{} + err = json.Unmarshal(content, &result) + if err != nil { + return 0, err + } + + return result.ClusterId, nil +} + +func (o *DiscoverEtcdInitialClusterOptions) findLocalClusterIdFromWal() (uint64, error) { + x, err := readClusterIdFromWAL(zap.NewNop(), o.DataDir) + return uint64(x), err +} + +func (o *DiscoverEtcdInitialClusterOptions) getClient() (*clientv3.Client, error) { + dialOptions := []grpc.DialOption{ + grpc.WithBlock(), // block until the underlying connection is up + } + + tlsInfo := transport.TLSInfo{ + CertFile: o.ClientCertFile, + KeyFile: o.ClientKeyFile, + TrustedCAFile: o.CABundleFile, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, err + } + + cfg := &clientv3.Config{ + DialOptions: dialOptions, + Endpoints: o.Endpoints, + DialTimeout: 2 * time.Second, // fail fast + TLS: tlsConfig, + } + + return clientv3.New(*cfg) +} + +func archiveDataDir(dataDir string) (string, error) { + // for testing + if strings.HasPrefix(dataDir, "/tmp") { + return "/tmp-removed-archive", nil + } + sourceDir := filepath.Join(dataDir, "member") + targetDir := filepath.Join(sourceDir + "-removed-archive-" + time.Now().Format("2006-01-02-030405")) + + fmt.Fprintf(os.Stderr, "attempting to archive %s to %s", sourceDir, targetDir) + if err := os.Rename(sourceDir, targetDir); err != nil { + return "", err + } + fmt.Fprintf(os.Stdout, "moved datadir successfully to %s\n", targetDir) + return targetDir, nil +} + +func stringifyMember(member *etcdserverpb.Member) string { + return fmt.Sprintf("{name=%q, peerURLs=[%s}, clientURLs=[%s]", member.Name, strings.Join(member.PeerURLs, ","), strings.Join(member.ClientURLs, ",")) +} + +// checkTargetMember populates the target member if it is part of the member list and print member details into etcd log. +func checkTargetMember(target url.URL, members []*etcdserverpb.Member) (*etcdserverpb.Member, bool) { + for _, member := range members { + for _, peerURL := range member.PeerURLs { + if peerURL == target.String() { + fmt.Fprintf(os.Stderr, " target=%s\n", stringifyMember(member)) + return member, true + } + } + } + return nil, false +} + +// logCurrentMembership prints the current etcd membership to the etcd logs. +func logCurrentMembership(members []*etcdserverpb.Member) { + for _, member := range members { + fmt.Fprintf(os.Stderr, " member=%s\n", stringifyMember(member)) + } + return +} + +// formatInitialCluster populates the initial cluster comma delimited string in the format =. +func formatInitialCluster(targetName string, target *etcdserverpb.Member, members []*etcdserverpb.Member) string { + var initialCluster []string + for _, member := range members { + if member.Name == "" { // this is the signal for whether or not a given peer is started + continue + } + for _, peerURL := range member.PeerURLs { + initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", member.Name, peerURL)) + } + } + if target.Name == "" { + // Adding unstarted member to the end of list + initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", targetName, target.PeerURLs[0])) + } + + return strings.Join(initialCluster, ",") +} diff --git a/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go new file mode 100644 index 000000000000..e6128be5af1d --- /dev/null +++ b/openshift-tools/pkg/discover-etcd-initial-cluster/initial-cluster_test.go @@ -0,0 +1,113 @@ +package discover_etcd_initial_cluster + +import ( + "regexp" + "testing" + + "go.etcd.io/etcd/api/v3/etcdserverpb" +) + +var ( + emptyInitialCluster = "" + startedEtcdMember = &etcdserverpb.Member{Name: "etcd-0", PeerURLs: []string{"https://etcd-0:2380"}} + unstartedEtcdMember = &etcdserverpb.Member{Name: "", PeerURLs: []string{"https://etcd-0:2380"}} + notFoundEtcdMember = &etcdserverpb.Member{Name: "not-found", PeerURLs: []string{"https://not-found:2380"}} +) + +func Test_ensureValidMember(t *testing.T) { + tests := map[string]struct { + member *etcdserverpb.Member + dataDirExists bool + wantMemberFound bool + mismatchingClusterId bool + wantInitialCluster string + wantErr bool + wantErrString string + }{ + "started member found no dataDir": { + member: startedEtcdMember, + wantMemberFound: true, + dataDirExists: false, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "dataDir has been destroyed and must be removed from the cluster", + }, + "started member found with dataDir": { + member: startedEtcdMember, + wantMemberFound: true, + dataDirExists: true, + wantInitialCluster: emptyInitialCluster, + wantErr: false, + }, + "member not found with dataDir": { + member: notFoundEtcdMember, + wantMemberFound: false, + dataDirExists: true, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "check operator logs for possible scaling problems", + }, + "member not found with dataDir and mismatching clusterid": { + member: notFoundEtcdMember, + wantMemberFound: true, + mismatchingClusterId: true, + dataDirExists: true, + wantInitialCluster: "", + wantErr: false, + }, + "member not found no dataDir": { + member: notFoundEtcdMember, + wantMemberFound: false, + dataDirExists: false, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "check operator logs for possible scaling problems", + }, + "unstarted member found with dataDir": { + member: unstartedEtcdMember, + wantMemberFound: true, + dataDirExists: true, + wantInitialCluster: emptyInitialCluster, + wantErr: true, + wantErrString: "previous members dataDir exists: archiving", + }, + "unstarted member found no dataDir": { + member: unstartedEtcdMember, + wantMemberFound: true, + dataDirExists: false, + wantInitialCluster: "etcd-0=https://etcd-0:2380", + wantErr: false, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + o := DiscoverEtcdInitialClusterOptions{ + TargetPeerURLHost: "etcd-0", + TargetPeerURLScheme: "https", + TargetPeerURLPort: "2380", + TargetName: "etcd-0", + DataDir: "/tmp", + } + gotInitialCluster, gotMemberFound, err := o.getInitialCluster([]*etcdserverpb.Member{test.member}, test.dataDirExists, test.mismatchingClusterId) + if gotInitialCluster != test.wantInitialCluster { + t.Fatalf("initialCluster: want: %q, got: %q", test.wantInitialCluster, gotInitialCluster) + } + if err != nil && !test.wantErr { + t.Fatalf("unexpected error: %v", err) + } + if err == nil && test.wantErr { + t.Fatal("expected error got nil") + } + if gotMemberFound != test.wantMemberFound { + t.Fatalf("memberFound: want %v, got %v", gotMemberFound, test.wantMemberFound) + } + if test.wantErrString != "" { + regex := regexp.MustCompile(test.wantErrString) + if len(regex.FindAll([]byte(err.Error()), -1)) != 1 { + t.Fatalf("unexpected error wanted %q in %q", test.wantErrString, err.Error()) + } + } + }) + } + +} diff --git a/openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go b/openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go new file mode 100644 index 000000000000..a7695b207112 --- /dev/null +++ b/openshift-tools/pkg/discover-etcd-initial-cluster/walutil.go @@ -0,0 +1,58 @@ +package discover_etcd_initial_cluster + +import ( + "errors" + "go.etcd.io/etcd/server/v3/datadir" + "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "path/filepath" + + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/pkg/v3/pbutil" + "go.etcd.io/etcd/server/v3/wal" + "go.etcd.io/etcd/server/v3/wal/walpb" + + "go.uber.org/zap" +) + +func readClusterIdFromWAL(lg *zap.Logger, dataDir string) (cid types.ID, err error) { + walDir := datadir.ToWalDir(dataDir) + snapDir := filepath.Join(datadir.ToMemberDir(dataDir), "snap") + + // Find a snapshot to start/restart a raft node + ss := snap.New(lg, snapDir) + + var walSnaps []walpb.Snapshot + walSnaps, err = wal.ValidSnapshotEntries(lg, walDir) + if err != nil { + return 0, err + } + + snapshot, err := ss.LoadNewestAvailable(walSnaps) + if err != nil && !errors.Is(err, snap.ErrNoSnapshot) { + return 0, err + } + + var walSnap walpb.Snapshot + if snapshot != nil { + walSnap.Index, walSnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term + } + + w, err := wal.Open(lg, walDir, walSnap) + if err != nil { + return 0, err + } + + defer func() { + err = errors.Join(err, w.Close()) + }() + + walMeta, _, _, err := w.ReadAll() + if err != nil { + return 0, err + } + + var metadata pb.Metadata + pbutil.MustUnmarshal(&metadata, walMeta) + return types.ID(metadata.ClusterID), nil +} diff --git a/pkg/adt/interval_tree.go b/pkg/adt/interval_tree.go index 74a9aeb141e7..271e3d389344 100644 --- a/pkg/adt/interval_tree.go +++ b/pkg/adt/interval_tree.go @@ -241,34 +241,34 @@ type intervalTree struct { // // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.4, p324 // -// 0. RB-DELETE(T, z) -// 1. -// 2. y = z -// 3. y-original-color = y.color -// 4. -// 5. if z.left == T.nil -// 6. x = z.right -// 7. RB-TRANSPLANT(T, z, z.right) -// 8. else if z.right == T.nil -// 9. x = z.left -// 10. RB-TRANSPLANT(T, z, z.left) -// 11. else -// 12. y = TREE-MINIMUM(z.right) -// 13. y-original-color = y.color -// 14. x = y.right -// 15. if y.p == z -// 16. x.p = y -// 17. else -// 18. RB-TRANSPLANT(T, y, y.right) -// 19. y.right = z.right -// 20. y.right.p = y -// 21. RB-TRANSPLANT(T, z, y) -// 22. y.left = z.left -// 23. y.left.p = y -// 24. y.color = z.color -// 25. -// 26. if y-original-color == BLACK -// 27. RB-DELETE-FIXUP(T, x) +// RB-DELETE(T, z) +// +// y = z +// y-original-color = y.color +// +// if z.left == T.nil +// x = z.right +// RB-TRANSPLANT(T, z, z.right) +// else if z.right == T.nil +// x = z.left +// RB-TRANSPLANT(T, z, z.left) +// else +// y = TREE-MINIMUM(z.right) +// y-original-color = y.color +// x = y.right +// if y.p == z +// x.p = y +// else +// RB-TRANSPLANT(T, y, y.right) +// y.right = z.right +// y.right.p = y +// RB-TRANSPLANT(T, z, y) +// y.left = z.left +// y.left.p = y +// y.color = z.color +// +// if y-original-color == BLACK +// RB-DELETE-FIXUP(T, x) // Delete removes the node with the given interval from the tree, returning // true if a node is in fact removed. @@ -317,48 +317,47 @@ func (ivt *intervalTree) Delete(ivl Interval) bool { // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.4, p326 // -// 0. RB-DELETE-FIXUP(T, z) -// 1. -// 2. while x ≠ T.root and x.color == BLACK -// 3. if x == x.p.left -// 4. w = x.p.right -// 5. if w.color == RED -// 6. w.color = BLACK -// 7. x.p.color = RED -// 8. LEFT-ROTATE(T, x, p) -// 9. if w.left.color == BLACK and w.right.color == BLACK -// 10. w.color = RED -// 11. x = x.p -// 12. else if w.right.color == BLACK -// 13. w.left.color = BLACK -// 14. w.color = RED -// 15. RIGHT-ROTATE(T, w) -// 16. w = w.p.right -// 17. w.color = x.p.color -// 18. x.p.color = BLACK -// 19. LEFT-ROTATE(T, w.p) -// 20. x = T.root -// 21. else -// 22. w = x.p.left -// 23. if w.color == RED -// 24. w.color = BLACK -// 25. x.p.color = RED -// 26. RIGHT-ROTATE(T, x, p) -// 27. if w.right.color == BLACK and w.left.color == BLACK -// 28. w.color = RED -// 29. x = x.p -// 30. else if w.left.color == BLACK -// 31. w.right.color = BLACK -// 32. w.color = RED -// 33. LEFT-ROTATE(T, w) -// 34. w = w.p.left -// 35. w.color = x.p.color -// 36. x.p.color = BLACK -// 37. RIGHT-ROTATE(T, w.p) -// 38. x = T.root -// 39. -// 40. x.color = BLACK +// RB-DELETE-FIXUP(T, z) // +// while x ≠ T.root and x.color == BLACK +// if x == x.p.left +// w = x.p.right +// if w.color == RED +// w.color = BLACK +// x.p.color = RED +// LEFT-ROTATE(T, x, p) +// if w.left.color == BLACK and w.right.color == BLACK +// w.color = RED +// x = x.p +// else if w.right.color == BLACK +// w.left.color = BLACK +// w.color = RED +// RIGHT-ROTATE(T, w) +// w = w.p.right +// w.color = x.p.color +// x.p.color = BLACK +// LEFT-ROTATE(T, w.p) +// x = T.root +// else +// w = x.p.left +// if w.color == RED +// w.color = BLACK +// x.p.color = RED +// RIGHT-ROTATE(T, x, p) +// if w.right.color == BLACK and w.left.color == BLACK +// w.color = RED +// x = x.p +// else if w.left.color == BLACK +// w.right.color = BLACK +// w.color = RED +// LEFT-ROTATE(T, w) +// w = w.p.left +// w.color = x.p.color +// x.p.color = BLACK +// RIGHT-ROTATE(T, w.p) +// x = T.root +// +// x.color = BLACK func (ivt *intervalTree) deleteFixup(x *intervalNode) { for x != ivt.root && x.color(ivt.sentinel) == black { if x == x.parent.left { // line 3-20 @@ -439,32 +438,32 @@ func (ivt *intervalTree) createIntervalNode(ivl Interval, val interface{}) *inte // // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.3, p315 // -// 0. RB-INSERT(T, z) -// 1. -// 2. y = T.nil -// 3. x = T.root -// 4. -// 5. while x ≠ T.nil -// 6. y = x -// 7. if z.key < x.key -// 8. x = x.left -// 9. else -// 10. x = x.right -// 11. -// 12. z.p = y -// 13. -// 14. if y == T.nil -// 15. T.root = z -// 16. else if z.key < y.key -// 17. y.left = z -// 18. else -// 19. y.right = z -// 20. -// 21. z.left = T.nil -// 22. z.right = T.nil -// 23. z.color = RED -// 24. -// 25. RB-INSERT-FIXUP(T, z) +// RB-INSERT(T, z) +// +// y = T.nil +// x = T.root +// +// while x ≠ T.nil +// y = x +// if z.key < x.key +// x = x.left +// else +// x = x.right +// +// z.p = y +// +// if y == T.nil +// T.root = z +// else if z.key < y.key +// y.left = z +// else +// y.right = z +// +// z.left = T.nil +// z.right = T.nil +// z.color = RED +// +// RB-INSERT-FIXUP(T, z) // Insert adds a node with the given interval into the tree. func (ivt *intervalTree) Insert(ivl Interval, val interface{}) { @@ -499,38 +498,37 @@ func (ivt *intervalTree) Insert(ivl Interval, val interface{}) { // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.3, p316 // -// 0. RB-INSERT-FIXUP(T, z) -// 1. -// 2. while z.p.color == RED -// 3. if z.p == z.p.p.left -// 4. y = z.p.p.right -// 5. if y.color == RED -// 6. z.p.color = BLACK -// 7. y.color = BLACK -// 8. z.p.p.color = RED -// 9. z = z.p.p -// 10. else if z == z.p.right -// 11. z = z.p -// 12. LEFT-ROTATE(T, z) -// 13. z.p.color = BLACK -// 14. z.p.p.color = RED -// 15. RIGHT-ROTATE(T, z.p.p) -// 16. else -// 17. y = z.p.p.left -// 18. if y.color == RED -// 19. z.p.color = BLACK -// 20. y.color = BLACK -// 21. z.p.p.color = RED -// 22. z = z.p.p -// 23. else if z == z.p.right -// 24. z = z.p -// 25. RIGHT-ROTATE(T, z) -// 26. z.p.color = BLACK -// 27. z.p.p.color = RED -// 28. LEFT-ROTATE(T, z.p.p) -// 29. -// 30. T.root.color = BLACK +// RB-INSERT-FIXUP(T, z) // +// while z.p.color == RED +// if z.p == z.p.p.left +// y = z.p.p.right +// if y.color == RED +// z.p.color = BLACK +// y.color = BLACK +// z.p.p.color = RED +// z = z.p.p +// else if z == z.p.right +// z = z.p +// LEFT-ROTATE(T, z) +// z.p.color = BLACK +// z.p.p.color = RED +// RIGHT-ROTATE(T, z.p.p) +// else +// y = z.p.p.left +// if y.color == RED +// z.p.color = BLACK +// y.color = BLACK +// z.p.p.color = RED +// z = z.p.p +// else if z == z.p.right +// z = z.p +// RIGHT-ROTATE(T, z) +// z.p.color = BLACK +// z.p.p.color = RED +// LEFT-ROTATE(T, z.p.p) +// +// T.root.color = BLACK func (ivt *intervalTree) insertFixup(z *intervalNode) { for z.parent.color(ivt.sentinel) == red { if z.parent == z.parent.parent.left { // line 3-15 @@ -578,26 +576,25 @@ func (ivt *intervalTree) insertFixup(z *intervalNode) { // // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.2, p313 // -// 0. LEFT-ROTATE(T, x) -// 1. -// 2. y = x.right -// 3. x.right = y.left -// 4. -// 5. if y.left ≠ T.nil -// 6. y.left.p = x -// 7. -// 8. y.p = x.p -// 9. -// 10. if x.p == T.nil -// 11. T.root = y -// 12. else if x == x.p.left -// 13. x.p.left = y -// 14. else -// 15. x.p.right = y -// 16. -// 17. y.left = x -// 18. x.p = y +// LEFT-ROTATE(T, x) +// +// y = x.right +// x.right = y.left +// +// if y.left ≠ T.nil +// y.left.p = x +// +// y.p = x.p +// +// if x.p == T.nil +// T.root = y +// else if x == x.p.left +// x.p.left = y +// else +// x.p.right = y // +// y.left = x +// x.p = y func (ivt *intervalTree) rotateLeft(x *intervalNode) { // rotateLeft x must have right child if x.right == ivt.sentinel { @@ -624,26 +621,25 @@ func (ivt *intervalTree) rotateLeft(x *intervalNode) { // rotateRight moves x so it is right of its left child // -// 0. RIGHT-ROTATE(T, x) -// 1. -// 2. y = x.left -// 3. x.left = y.right -// 4. -// 5. if y.right ≠ T.nil -// 6. y.right.p = x -// 7. -// 8. y.p = x.p -// 9. -// 10. if x.p == T.nil -// 11. T.root = y -// 12. else if x == x.p.right -// 13. x.p.right = y -// 14. else -// 15. x.p.left = y -// 16. -// 17. y.right = x -// 18. x.p = y +// RIGHT-ROTATE(T, x) +// +// y = x.left +// x.left = y.right +// +// if y.right ≠ T.nil +// y.right.p = x +// +// y.p = x.p +// +// if x.p == T.nil +// T.root = y +// else if x == x.p.right +// x.p.right = y +// else +// x.p.left = y // +// y.right = x +// x.p = y func (ivt *intervalTree) rotateRight(x *intervalNode) { // rotateRight x must have left child if x.left == ivt.sentinel { diff --git a/pkg/adt/interval_tree_test.go b/pkg/adt/interval_tree_test.go index 608841f92f3a..1e1a05a6452e 100644 --- a/pkg/adt/interval_tree_test.go +++ b/pkg/adt/interval_tree_test.go @@ -63,27 +63,28 @@ func TestIntervalTreeInsert(t *testing.T) { // Use https://www.cs.usfca.edu/~galles/visualization/RedBlack.html for test case creation. // // Regular Binary Search Tree -// [0,1] -// \ -// [1,2] -// \ -// [3,4] -// \ -// [5,6] -// \ -// [7,8] -// \ -// [8,9] +// +// [0,1] +// \ +// [1,2] +// \ +// [3,4] +// \ +// [5,6] +// \ +// [7,8] +// \ +// [8,9] // // Self-Balancing Binary Search Tree -// [1,2] -// / \ -// [0,1] [5,6] -// / \ -// [3,4] [7,8] -// \ -// [8,9] // +// [1,2] +// / \ +// [0,1] [5,6] +// / \ +// [3,4] [7,8] +// \ +// [8,9] func TestIntervalTreeSelfBalanced(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewInt64Interval(0, 1), 0) @@ -120,58 +121,56 @@ func TestIntervalTreeSelfBalanced(t *testing.T) { // Use https://www.cs.usfca.edu/~galles/visualization/RedBlack.html for test case creation. // See https://github.com/etcd-io/etcd/issues/10877 for more detail. // -// // After insertion: -// [510,511] -// / \ -// ---------- ----------------------- -// / \ -// [82,83] [830,831] -// / \ / \ -// / \ / \ -// [11,12] [383,384](red) [647,648] [899,900](red) -// / \ / \ / \ -// / \ / \ / \ -// [261,262] [410,411] [514,515](red) [815,816](red) [888,889] [972,973] -// / \ / -// / \ / -// [238,239](red) [292,293](red) [953,954](red) // +// [510,511] +// / \ +// ---------- ----------------------- +// / \ +// [82,83] [830,831] +// / \ / \ +// / \ / \ +// [11,12] [383,384](red) [647,648] [899,900](red) +// / \ / \ / \ +// / \ / \ / \ +// [261,262] [410,411] [514,515](red) [815,816](red) [888,889] [972,973] +// / \ / +// / \ / +// [238,239](red) [292,293](red) [953,954](red) // // After deleting 514 (no rebalance): -// [510,511] -// / \ -// ---------- ----------------------- -// / \ -// [82,83] [830,831] -// / \ / \ -// / \ / \ -// [11,12] [383,384](red) [647,648] [899,900](red) -// / \ \ / \ -// / \ \ / \ -// [261,262] [410,411] [815,816](red) [888,889] [972,973] -// / \ / -// / \ / -// [238,239](red) [292,293](red) [953,954](red) // +// [510,511] +// / \ +// ---------- ----------------------- +// / \ +// [82,83] [830,831] +// / \ / \ +// / \ / \ +// [11,12] [383,384](red) [647,648] [899,900](red) +// / \ \ / \ +// / \ \ / \ +// [261,262] [410,411] [815,816](red) [888,889] [972,973] +// / \ / +// / \ / +// [238,239](red) [292,293](red) [953,954](red) // // After deleting 11 (requires rebalancing): -// [510,511] -// / \ -// ---------- -------------------------- -// / \ -// [383,384] [830,831] -// / \ / \ -// / \ / \ -// [261,262](red) [410,411] [647,648] [899,900](red) -// / \ \ / \ -// / \ \ / \ -// [82,83] [292,293] [815,816](red) [888,889] [972,973] -// \ / -// \ / -// [238,239](red) [953,954](red) -// // +// [510,511] +// / \ +// ---------- -------------------------- +// / \ +// [383,384] [830,831] +// / \ / \ +// / \ / \ +// [261,262](red) [410,411] [647,648] [899,900](red) +// / \ \ / \ +// / \ \ / \ +// [82,83] [292,293] [815,816](red) [888,889] [972,973] +// \ / +// \ / +// [238,239](red) [953,954](red) func TestIntervalTreeDelete(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewInt64Interval(510, 511), 0) diff --git a/pkg/expect/expect.go b/pkg/expect/expect.go index 12f95f98d9ff..95bc30823f29 100644 --- a/pkg/expect/expect.go +++ b/pkg/expect/expect.go @@ -25,6 +25,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/creack/pty" ) @@ -36,7 +37,6 @@ type ExpectProcess struct { fpty *os.File wg sync.WaitGroup - cond *sync.Cond // for broadcasting updates are available mu sync.Mutex // protects lines and err lines []string count int // increment whenever new line gets added @@ -60,7 +60,6 @@ func NewExpectWithEnv(name string, args []string, env []string) (ep *ExpectProce cmd: cmd, StopSignal: syscall.SIGKILL, } - ep.cond = sync.NewCond(&ep.mu) ep.cmd.Stderr = ep.cmd.Stdout ep.cmd.Stdin = nil @@ -77,52 +76,56 @@ func (ep *ExpectProcess) read() { defer ep.wg.Done() printDebugLines := os.Getenv("EXPECT_DEBUG") != "" r := bufio.NewReader(ep.fpty) - for ep.err == nil { - l, rerr := r.ReadString('\n') + for { + l, err := r.ReadString('\n') ep.mu.Lock() - ep.err = rerr if l != "" { if printDebugLines { fmt.Printf("%s-%d: %s", ep.cmd.Path, ep.cmd.Process.Pid, l) } ep.lines = append(ep.lines, l) ep.count++ - if len(ep.lines) == 1 { - ep.cond.Signal() - } + } + if err != nil { + ep.err = err + ep.mu.Unlock() + break } ep.mu.Unlock() } - ep.cond.Signal() } // ExpectFunc returns the first line satisfying the function f. func (ep *ExpectProcess) ExpectFunc(f func(string) bool) (string, error) { - lastLinesBuffer := make([]string, 0) + i := 0 - ep.mu.Lock() for { - for len(ep.lines) == 0 && ep.err == nil { - ep.cond.Wait() - } - if len(ep.lines) == 0 { - break - } - l := ep.lines[0] - ep.lines = ep.lines[1:] - lastLinesBuffer = append(lastLinesBuffer, l) - if l := len(lastLinesBuffer); l > DEBUG_LINES_TAIL { - lastLinesBuffer = lastLinesBuffer[l-DEBUG_LINES_TAIL : l-1] + ep.mu.Lock() + for i < len(ep.lines) { + line := ep.lines[i] + i++ + if f(line) { + ep.mu.Unlock() + return line, nil + } } - if f(l) { + if ep.err != nil { ep.mu.Unlock() - return l, nil + break } + ep.mu.Unlock() + time.Sleep(time.Millisecond * 100) + } + ep.mu.Lock() + lastLinesIndex := len(ep.lines) - DEBUG_LINES_TAIL + if lastLinesIndex < 0 { + lastLinesIndex = 0 } + lastLines := strings.Join(ep.lines[lastLinesIndex:], "") ep.mu.Unlock() return "", fmt.Errorf("match not found."+ " Set EXPECT_DEBUG for more info Err: %v, last lines:\n%s", - ep.err, strings.Join(lastLinesBuffer, "")) + ep.err, lastLines) } // Expect returns the first line containing the given string. @@ -146,6 +149,11 @@ func (ep *ExpectProcess) Signal(sig os.Signal) error { return ep.cmd.Process.Signal(sig) } +// Wait waits for the process to finish. +func (ep *ExpectProcess) Wait() { + ep.wg.Wait() +} + // Close waits for the expect process to exit. // Close currently does not return error if process exited with !=0 status. // TODO: Close should expose underlying proces failure by default. @@ -189,3 +197,13 @@ func (ep *ExpectProcess) ProcessError() error { } return ep.err } + +func (ep *ExpectProcess) Lines() []string { + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.lines +} + +func (ep *ExpectProcess) IsRunning() bool { + return ep.cmd != nil +} diff --git a/pkg/flags/uint32.go b/pkg/flags/uint32.go new file mode 100644 index 000000000000..496730a45499 --- /dev/null +++ b/pkg/flags/uint32.go @@ -0,0 +1,45 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flags + +import ( + "flag" + "strconv" +) + +type uint32Value uint32 + +// NewUint32Value creates an uint32 instance with the provided value. +func NewUint32Value(v uint32) *uint32Value { + val := new(uint32Value) + *val = uint32Value(v) + return val +} + +// Set parses a command line uint32 value. +// Implements "flag.Value" interface. +func (i *uint32Value) Set(s string) error { + v, err := strconv.ParseUint(s, 0, 32) + *i = uint32Value(v) + return err +} + +func (i *uint32Value) String() string { return strconv.FormatUint(uint64(*i), 10) } + +// Uint32FromFlag return the uint32 value of a flag with the given name +func Uint32FromFlag(fs *flag.FlagSet, name string) uint32 { + val := *fs.Lookup(name).Value.(*uint32Value) + return uint32(val) +} diff --git a/pkg/flags/uint32_test.go b/pkg/flags/uint32_test.go new file mode 100644 index 000000000000..aa7487a23208 --- /dev/null +++ b/pkg/flags/uint32_test.go @@ -0,0 +1,111 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flags + +import ( + "flag" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUint32Value(t *testing.T) { + cases := []struct { + name string + s string + expectedVal uint32 + expectError bool + }{ + { + name: "normal uint32 value", + s: "200", + expectedVal: 200, + }, + { + name: "zero value", + s: "0", + expectedVal: 0, + }, + { + name: "negative int value", + s: "-200", + expectError: true, + }, + { + name: "invalid integer value", + s: "invalid", + expectError: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var val uint32Value + err := val.Set(tc.s) + + if tc.expectError { + if err == nil { + t.Errorf("Expected failure on parsing uint32 value from %s", tc.s) + } + } else { + if err != nil { + t.Errorf("Unexpected error when parsing %s: %v", tc.s, err) + } + assert.Equal(t, uint32(val), tc.expectedVal) + } + }) + } +} + +func TestUint32FromFlag(t *testing.T) { + const flagName = "max-concurrent-streams" + + cases := []struct { + name string + defaultVal uint32 + arguments []string + expectedVal uint32 + }{ + { + name: "only default value", + defaultVal: 15, + arguments: []string{}, + expectedVal: 15, + }, + { + name: "argument has different value from the default one", + defaultVal: 16, + arguments: []string{"--max-concurrent-streams", "200"}, + expectedVal: 200, + }, + { + name: "argument has the same value from the default one", + defaultVal: 105, + arguments: []string{"--max-concurrent-streams", "105"}, + expectedVal: 105, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fs := flag.NewFlagSet("etcd", flag.ContinueOnError) + fs.Var(NewUint32Value(tc.defaultVal), flagName, "Maximum concurrent streams that each client can open at a time.") + if err := fs.Parse(tc.arguments); err != nil { + t.Fatalf("Unexpected error: %v\n", err) + } + actualMaxStream := Uint32FromFlag(fs, flagName) + assert.Equal(t, actualMaxStream, tc.expectedVal) + }) + } +} diff --git a/pkg/flags/unique_urls.go b/pkg/flags/unique_urls.go index 5b22ef21ad77..cc9b13294b31 100644 --- a/pkg/flags/unique_urls.go +++ b/pkg/flags/unique_urls.go @@ -50,7 +50,11 @@ func (us *UniqueURLs) Set(s string) error { us.Values = make(map[string]struct{}) us.uss = make([]url.URL, 0) for _, v := range ss { - us.Values[v.String()] = struct{}{} + x := v.String() + if _, exists := us.Values[x]; exists { + continue + } + us.Values[x] = struct{}{} us.uss = append(us.uss, v) } return nil diff --git a/pkg/flags/unique_urls_test.go b/pkg/flags/unique_urls_test.go index adc4a6b5a19b..2d41ff35aabe 100644 --- a/pkg/flags/unique_urls_test.go +++ b/pkg/flags/unique_urls_test.go @@ -15,8 +15,11 @@ package flags import ( - "reflect" + "flag" + "strings" "testing" + + "github.com/stretchr/testify/require" ) func TestNewUniqueURLsWithExceptions(t *testing.T) { @@ -83,11 +86,29 @@ func TestNewUniqueURLsWithExceptions(t *testing.T) { } for i := range tests { uv := NewUniqueURLsWithExceptions(tests[i].s, tests[i].exception) - if !reflect.DeepEqual(tests[i].exp, uv.Values) { - t.Fatalf("#%d: expected %+v, got %+v", i, tests[i].exp, uv.Values) - } - if uv.String() != tests[i].rs { - t.Fatalf("#%d: expected %q, got %q", i, tests[i].rs, uv.String()) - } + require.Equal(t, tests[i].exp, uv.Values) + require.Equal(t, uv.String(), tests[i].rs) + } +} + +func TestUniqueURLsFromFlag(t *testing.T) { + const name = "test" + urls := []string{ + "https://1.2.3.4:1", + "https://1.2.3.4:2", + "https://1.2.3.4:3", + "https://1.2.3.4:1", + } + fs := flag.NewFlagSet(name, flag.ExitOnError) + u := NewUniqueURLsWithExceptions(strings.Join(urls, ",")) + fs.Var(u, name, "usage") + uss := UniqueURLsFromFlag(fs, name) + + require.Equal(t, len(u.Values), len(uss)) + + um := make(map[string]struct{}) + for _, x := range uss { + um[x.String()] = struct{}{} } + require.Equal(t, u.Values, um) } diff --git a/pkg/flags/urls_test.go b/pkg/flags/urls_test.go index ff4bda8d407c..ebc9a2674102 100644 --- a/pkg/flags/urls_test.go +++ b/pkg/flags/urls_test.go @@ -29,9 +29,6 @@ func TestValidateURLsValueBad(t *testing.T) { // bad port specification "127.0.0.1:foo", "127.0.0.1:", - // unix sockets not supported - "unix://", - "unix://tmp/etcd.sock", // bad strings "somewhere", "234#$", @@ -56,6 +53,9 @@ func TestNewURLsValue(t *testing.T) { {s: "http://10.1.1.1:80", exp: []url.URL{{Scheme: "http", Host: "10.1.1.1:80"}}}, {s: "http://localhost:80", exp: []url.URL{{Scheme: "http", Host: "localhost:80"}}}, {s: "http://:80", exp: []url.URL{{Scheme: "http", Host: ":80"}}}, + {s: "unix://tmp/etcd.sock", exp: []url.URL{{Scheme: "unix", Host: "tmp", Path: "/etcd.sock"}}}, + {s: "unix:///tmp/127.27.84.4:23432", exp: []url.URL{{Scheme: "unix", Path: "/tmp/127.27.84.4:23432"}}}, + {s: "unix://127.0.0.5:1456", exp: []url.URL{{Scheme: "unix", Host: "127.0.0.5:1456"}}}, { s: "http://localhost:1,https://localhost:2", exp: []url.URL{ diff --git a/pkg/go.mod b/pkg/go.mod index afb2b9e19236..092c74cdbf41 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -1,17 +1,33 @@ module go.etcd.io/etcd/pkg/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 require ( github.com/creack/pty v1.1.11 github.com/dustin/go-humanize v1.0.0 - github.com/golang/protobuf v1.5.1 // indirect github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.0 - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 - go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 - google.golang.org/grpc v1.37.0 + github.com/stretchr/testify v1.9.0 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 + go.uber.org/zap v1.17.0 + google.golang.org/grpc v1.59.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( diff --git a/pkg/go.sum b/pkg/go.sum index a0c5c50c2b14..567c4e454551 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -23,15 +23,12 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -43,10 +40,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -55,7 +48,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -66,30 +58,18 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1 h1:jAbXjIeW2ZSW2AwFxlGTDoc2CjI2XujLkV3ArsZFCvc= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -162,7 +142,6 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -192,10 +171,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -209,8 +187,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19 h1:040c3dLNhgFQkoojH2AMpHCy4SrvhmxdU72d9GLGGE0= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -247,8 +225,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -270,12 +249,13 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -288,7 +268,6 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -297,8 +276,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -317,28 +294,15 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= @@ -348,15 +312,16 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/pkg/grpc_testing/recorder.go b/pkg/grpc_testing/recorder.go new file mode 100644 index 000000000000..d6b6d2aac2b2 --- /dev/null +++ b/pkg/grpc_testing/recorder.go @@ -0,0 +1,69 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc_testing + +import ( + "context" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +type GrpcRecorder struct { + mux sync.RWMutex + requests []RequestInfo +} + +type RequestInfo struct { + FullMethod string + Authority string +} + +func (ri *GrpcRecorder) UnaryInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ri.record(toRequestInfo(ctx, info)) + resp, err := handler(ctx, req) + return resp, err + } +} + +func (ri *GrpcRecorder) RecordedRequests() []RequestInfo { + ri.mux.RLock() + defer ri.mux.RUnlock() + reqs := make([]RequestInfo, len(ri.requests)) + copy(reqs, ri.requests) + return reqs +} + +func toRequestInfo(ctx context.Context, info *grpc.UnaryServerInfo) RequestInfo { + req := RequestInfo{ + FullMethod: info.FullMethod, + } + md, ok := metadata.FromIncomingContext(ctx) + if ok { + as := md.Get(":authority") + if len(as) != 0 { + req.Authority = as[0] + } + } + return req +} + +func (ri *GrpcRecorder) record(r RequestInfo) { + ri.mux.Lock() + defer ri.mux.Unlock() + ri.requests = append(ri.requests, r) +} diff --git a/pkg/grpc_testing/stub_server.go b/pkg/grpc_testing/stub_server.go index e7e8c49d4951..92e0788a0920 100644 --- a/pkg/grpc_testing/stub_server.go +++ b/pkg/grpc_testing/stub_server.go @@ -6,7 +6,7 @@ import ( "net" "google.golang.org/grpc" - testpb "google.golang.org/grpc/test/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) // StubServer is borrowed from the interal package of grpc-go. diff --git a/pkg/ioutil/pagewriter.go b/pkg/ioutil/pagewriter.go index cf9a8dc664dc..62eb5cd43e50 100644 --- a/pkg/ioutil/pagewriter.go +++ b/pkg/ioutil/pagewriter.go @@ -15,6 +15,7 @@ package ioutil import ( + "fmt" "io" ) @@ -41,6 +42,9 @@ type PageWriter struct { // NewPageWriter creates a new PageWriter. pageBytes is the number of bytes // to write per page. pageOffset is the starting offset of io.Writer. func NewPageWriter(w io.Writer, pageBytes, pageOffset int) *PageWriter { + if pageBytes <= 0 { + panic(fmt.Sprintf("assertion failed: invalid pageBytes (%d) value, it must be greater than 0", pageBytes)) + } return &PageWriter{ w: w, pageOffset: pageOffset, diff --git a/pkg/ioutil/pagewriter_test.go b/pkg/ioutil/pagewriter_test.go index 10610691c3c8..e05c71f7cb6e 100644 --- a/pkg/ioutil/pagewriter_test.go +++ b/pkg/ioutil/pagewriter_test.go @@ -17,6 +17,8 @@ package ioutil import ( "math/rand" "testing" + + "github.com/stretchr/testify/assert" ) func TestPageWriterRandom(t *testing.T) { @@ -37,14 +39,14 @@ func TestPageWriterRandom(t *testing.T) { if cw.writeBytes > n { t.Fatalf("wrote %d bytes to io.Writer, but only wrote %d bytes", cw.writeBytes, n) } - if n-cw.writeBytes > pageBytes { - t.Fatalf("got %d bytes pending, expected less than %d bytes", n-cw.writeBytes, pageBytes) + if maxPendingBytes := pageBytes + defaultBufferBytes; n-cw.writeBytes > maxPendingBytes { + t.Fatalf("got %d bytes pending, expected less than %d bytes", n-cw.writeBytes, maxPendingBytes) } t.Logf("total writes: %d", cw.writes) t.Logf("total write bytes: %d (of %d)", cw.writeBytes, n) } -// TestPageWriterPariallack tests the case where a write overflows the buffer +// TestPageWriterPartialSlack tests the case where a write overflows the buffer // but there is not enough data to complete the slack write. func TestPageWriterPartialSlack(t *testing.T) { defaultBufferBytes = 1024 @@ -111,6 +113,45 @@ func TestPageWriterOffset(t *testing.T) { } } +func TestPageWriterPageBytes(t *testing.T) { + cases := []struct { + name string + pageBytes int + expectPanic bool + }{ + { + name: "normal page bytes", + pageBytes: 4096, + expectPanic: false, + }, + { + name: "negative page bytes", + pageBytes: -1, + expectPanic: true, + }, + { + name: "zero page bytes", + pageBytes: 0, + expectPanic: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + defaultBufferBytes = 1024 + cw := &checkPageWriter{pageBytes: tc.pageBytes, t: t} + if tc.expectPanic { + assert.Panicsf(t, func() { + NewPageWriter(cw, tc.pageBytes, 0) + }, "expected panic when pageBytes is %d", tc.pageBytes) + } else { + pw := NewPageWriter(cw, tc.pageBytes, 0) + assert.NotEqual(t, pw, nil) + } + }) + } +} + // checkPageWriter implements an io.Writer that fails a test on unaligned writes. type checkPageWriter struct { pageBytes int diff --git a/pkg/netutil/netutil.go b/pkg/netutil/netutil.go index bf737a4d9426..689927b4c676 100644 --- a/pkg/netutil/netutil.go +++ b/pkg/netutil/netutil.go @@ -103,7 +103,7 @@ func resolveURL(ctx context.Context, lg *zap.Logger, u url.URL) (string, error) ) return "", err } - if host == "localhost" || net.ParseIP(host) != nil { + if host == "localhost" { return "", nil } for ctx.Err() == nil { @@ -148,20 +148,31 @@ func urlsEqual(ctx context.Context, lg *zap.Logger, a []url.URL, b []url.URL) (b if len(a) != len(b) { return false, fmt.Errorf("len(%q) != len(%q)", urlsToStrings(a), urlsToStrings(b)) } + + sort.Sort(types.URLs(a)) + sort.Sort(types.URLs(b)) + var needResolve bool + for i := range a { + if !reflect.DeepEqual(a[i], b[i]) { + needResolve = true + break + } + } + if !needResolve { + return true, nil + } + + // If URLs are not equal, try to resolve it and compare again. urls, err := resolveTCPAddrs(ctx, lg, [][]url.URL{a, b}) if err != nil { return false, err } - preva, prevb := a, b a, b = urls[0], urls[1] sort.Sort(types.URLs(a)) sort.Sort(types.URLs(b)) for i := range a { if !reflect.DeepEqual(a[i], b[i]) { - return false, fmt.Errorf("%q(resolved from %q) != %q(resolved from %q)", - a[i].String(), preva[i].String(), - b[i].String(), prevb[i].String(), - ) + return false, fmt.Errorf("resolved urls: %q != %q", a[i].String(), b[i].String()) } } return true, nil @@ -174,27 +185,13 @@ func URLStringsEqual(ctx context.Context, lg *zap.Logger, a []string, b []string if len(a) != len(b) { return false, fmt.Errorf("len(%q) != len(%q)", a, b) } - urlsA := make([]url.URL, 0) - for _, str := range a { - u, err := url.Parse(str) - if err != nil { - return false, fmt.Errorf("failed to parse %q", str) - } - urlsA = append(urlsA, *u) - } - urlsB := make([]url.URL, 0) - for _, str := range b { - u, err := url.Parse(str) - if err != nil { - return false, fmt.Errorf("failed to parse %q", str) - } - urlsB = append(urlsB, *u) + urlsA, err := stringsToURLs(a) + if err != nil { + return false, err } - if lg == nil { - lg, _ = zap.NewProduction() - if lg == nil { - lg = zap.NewExample() - } + urlsB, err := stringsToURLs(b) + if err != nil { + return false, err } return urlsEqual(ctx, lg, urlsA, urlsB) } @@ -207,6 +204,18 @@ func urlsToStrings(us []url.URL) []string { return rs } +func stringsToURLs(us []string) ([]url.URL, error) { + urls := make([]url.URL, 0, len(us)) + for _, str := range us { + u, err := url.Parse(str) + if err != nil { + return nil, fmt.Errorf("failed to parse string to URL: %q", str) + } + urls = append(urls, *u) + } + return urls, nil +} + func IsNetworkTimeoutError(err error) bool { nerr, ok := err.(net.Error) return ok && nerr.Timeout() diff --git a/pkg/netutil/netutil_test.go b/pkg/netutil/netutil_test.go index 42b05ca295ad..c085969c0b02 100644 --- a/pkg/netutil/netutil_test.go +++ b/pkg/netutil/netutil_test.go @@ -17,6 +17,7 @@ package netutil import ( "context" "errors" + "fmt" "net" "net/url" "reflect" @@ -119,13 +120,16 @@ func TestResolveTCPAddrs(t *testing.T) { if err != nil { return nil, err } - if tt.hostMap[host] == "" { - return nil, errors.New("cannot resolve host") - } i, err := strconv.Atoi(port) if err != nil { return nil, err } + if ip := net.ParseIP(host); ip != nil { + return &net.TCPAddr{IP: ip, Port: i, Zone: ""}, nil + } + if tt.hostMap[host] == "" { + return nil, errors.New("cannot resolve host") + } return &net.TCPAddr{IP: net.ParseIP(tt.hostMap[host]), Port: i, Zone: ""}, nil } ctx, cancel := context.WithTimeout(context.TODO(), time.Second) @@ -151,128 +155,151 @@ func TestURLsEqual(t *testing.T) { "second.com": "10.0.11.2", } resolveTCPAddr = func(ctx context.Context, addr string) (*net.TCPAddr, error) { - host, port, herr := net.SplitHostPort(addr) - if herr != nil { - return nil, herr - } - if _, ok := hostm[host]; !ok { - return nil, errors.New("cannot resolve host.") + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err } i, err := strconv.Atoi(port) if err != nil { return nil, err } + if ip := net.ParseIP(host); ip != nil { + return &net.TCPAddr{IP: ip, Port: i, Zone: ""}, nil + } + if hostm[host] == "" { + return nil, errors.New("cannot resolve host") + } return &net.TCPAddr{IP: net.ParseIP(hostm[host]), Port: i, Zone: ""}, nil } tests := []struct { + n int a []url.URL b []url.URL expect bool err error }{ { + n: 0, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, expect: true, }, { + n: 1, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}}, expect: true, }, { + n: 2, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "https", Host: "10.0.10.1:2379"}}, expect: false, - err: errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "https://10.0.10.1:2379"(resolved from "https://10.0.10.1:2379")`), + err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "https://10.0.10.1:2379"`), }, { + n: 3, a: []url.URL{{Scheme: "https", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}}, expect: false, - err: errors.New(`"https://10.0.10.1:2379"(resolved from "https://example.com:2379") != "http://10.0.10.1:2379"(resolved from "http://10.0.10.1:2379")`), + err: errors.New(`resolved urls: "https://10.0.10.1:2379" != "http://10.0.10.1:2379"`), }, { + n: 4, a: []url.URL{{Scheme: "unix", Host: "abc:2379"}}, b: []url.URL{{Scheme: "unix", Host: "abc:2379"}}, expect: true, }, { + n: 5, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: true, }, { + n: 6, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: true, }, { + n: 7, a: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: true, }, { + n: 8, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, - err: errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://127.0.0.1:2380"(resolved from "http://127.0.0.1:2380")`), + err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://127.0.0.1:2380"`), }, { + n: 9, a: []url.URL{{Scheme: "http", Host: "example.com:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}}, expect: false, - err: errors.New(`"http://10.0.10.1:2380"(resolved from "http://example.com:2380") != "http://10.0.10.1:2379"(resolved from "http://10.0.10.1:2379")`), + err: errors.New(`resolved urls: "http://10.0.10.1:2380" != "http://10.0.10.1:2379"`), }, { + n: 10, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, expect: false, - err: errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`), + err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://10.0.0.1:2379"`), }, { + n: 11, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, expect: false, - err: errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`), + err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "http://10.0.0.1:2379"`), }, { + n: 12, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, - err: errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://127.0.0.1:2380"(resolved from "http://127.0.0.1:2380")`), + err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://127.0.0.1:2380"`), }, { + n: 13, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, - err: errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "http://127.0.0.1:2380"(resolved from "http://127.0.0.1:2380")`), + err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "http://127.0.0.1:2380"`), }, { + n: 14, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, - err: errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`), + err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://10.0.0.1:2379"`), }, { + n: 15, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, - err: errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`), + err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "http://10.0.0.1:2379"`), }, { + n: 16, a: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`len(["http://10.0.0.1:2379"]) != len(["http://10.0.0.1:2379" "http://127.0.0.1:2380"])`), }, { + n: 17, a: []url.URL{{Scheme: "http", Host: "first.com:2379"}, {Scheme: "http", Host: "second.com:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.11.1:2379"}, {Scheme: "http", Host: "10.0.11.2:2380"}}, expect: true, }, { + n: 18, a: []url.URL{{Scheme: "http", Host: "second.com:2380"}, {Scheme: "http", Host: "first.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.11.1:2379"}, {Scheme: "http", Host: "10.0.11.2:2380"}}, expect: true, @@ -282,21 +309,48 @@ func TestURLsEqual(t *testing.T) { for i, test := range tests { result, err := urlsEqual(context.TODO(), zap.NewExample(), test.a, test.b) if result != test.expect { - t.Errorf("#%d: a:%v b:%v, expected %v but %v", i, test.a, test.b, test.expect, result) + t.Errorf("idx=%d #%d: a:%v b:%v, expected %v but %v", i, test.n, test.a, test.b, test.expect, result) } if test.err != nil { if err.Error() != test.err.Error() { - t.Errorf("#%d: err expected %v but %v", i, test.err, err) + t.Errorf("idx=%d #%d: err expected %v but %v", i, test.n, test.err, err) } } } } func TestURLStringsEqual(t *testing.T) { - result, err := URLStringsEqual(context.TODO(), zap.NewExample(), []string{"http://127.0.0.1:8080"}, []string{"http://127.0.0.1:8080"}) - if !result { - t.Errorf("unexpected result %v", result) + defer func() { resolveTCPAddr = resolveTCPAddrDefault }() + errOnResolve := func(ctx context.Context, addr string) (*net.TCPAddr, error) { + return nil, fmt.Errorf("unexpected attempt to resolve: %q", addr) + } + cases := []struct { + urlsA []string + urlsB []string + resolver func(ctx context.Context, addr string) (*net.TCPAddr, error) + }{ + {[]string{"http://127.0.0.1:8080"}, []string{"http://127.0.0.1:8080"}, resolveTCPAddrDefault}, + {[]string{ + "http://host1:8080", + "http://host2:8080", + }, []string{ + "http://host1:8080", + "http://host2:8080", + }, errOnResolve}, + { + urlsA: []string{"https://[c262:266f:fa53:0ee6:966e:e3f0:d68f:b046]:2380"}, + urlsB: []string{"https://[c262:266f:fa53:ee6:966e:e3f0:d68f:b046]:2380"}, + resolver: resolveTCPAddrDefault, + }, } - if err != nil { - t.Errorf("unexpected error %v", err) + for idx, c := range cases { + t.Logf("TestURLStringsEqual, case #%d", idx) + resolveTCPAddr = c.resolver + result, err := URLStringsEqual(context.TODO(), zap.NewExample(), c.urlsA, c.urlsB) + if !result { + t.Errorf("unexpected result %v", result) + } + if err != nil { + t.Errorf("unexpected error %v", err) + } } } diff --git a/pkg/proxy/server.go b/pkg/proxy/server.go index 9a7b105f9a79..72a0c7483d23 100644 --- a/pkg/proxy/server.go +++ b/pkg/proxy/server.go @@ -37,17 +37,8 @@ var ( defaultDialTimeout = 3 * time.Second defaultBufferSize = 48 * 1024 defaultRetryInterval = 10 * time.Millisecond - defaultLogger *zap.Logger ) -func init() { - var err error - defaultLogger, err = zap.NewProduction() - if err != nil { - panic(err) - } -} - // Server defines proxy server layer that simulates common network faults: // latency spikes and packet drop or corruption. The proxy overhead is very // small overhead (<500μs per request). Please run tests to compute actual @@ -240,9 +231,6 @@ func NewServer(cfg ServerConfig) Server { if s.retryInterval == 0 { s.retryInterval = defaultRetryInterval } - if s.lg == nil { - s.lg = defaultLogger - } close(s.pauseAcceptc) close(s.pauseTxc) diff --git a/pkg/stringutil/rand.go b/pkg/stringutil/rand.go index a15b0de0c08f..96d9df311cfb 100644 --- a/pkg/stringutil/rand.go +++ b/pkg/stringutil/rand.go @@ -24,7 +24,7 @@ func UniqueStrings(slen uint, n int) (ss []string) { exist := make(map[string]struct{}) ss = make([]string, 0, n) for len(ss) < n { - s := randString(slen) + s := RandString(slen) if _, ok := exist[s]; !ok { ss = append(ss, s) exist[s] = struct{}{} @@ -37,14 +37,14 @@ func UniqueStrings(slen uint, n int) (ss []string) { func RandomStrings(slen uint, n int) (ss []string) { ss = make([]string, 0, n) for i := 0; i < n; i++ { - ss = append(ss, randString(slen)) + ss = append(ss, RandString(slen)) } return ss } const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" -func randString(l uint) string { +func RandString(l uint) string { rand.Seed(time.Now().UnixNano()) s := make([]byte, l) for i := 0; i < int(l); i++ { diff --git a/raft/OWNERS b/raft/OWNERS index ab781066e237..35fc8f687134 100644 --- a/raft/OWNERS +++ b/raft/OWNERS @@ -1,19 +1,5 @@ approvers: -- heyitsanthony -- philips -- fanminshi -- gyuho -- mitake -- jpbetz -- xiang90 -- bdarnell + - hexfusion + - smarterclayton reviewers: -- heyitsanthony -- philips -- fanminshi -- gyuho -- mitake -- jpbetz -- xiang90 -- bdarnell -- tschottdorf + - hexfusion diff --git a/raft/confchange/confchange.go b/raft/confchange/confchange.go index 55e6830ce8ae..dddbcc9d9b57 100644 --- a/raft/confchange/confchange.go +++ b/raft/confchange/confchange.go @@ -37,9 +37,11 @@ type Changer struct { // config is empty and initializes it with a copy of the incoming (=left) // majority config. That is, it transitions from // -// (1 2 3)&&() +// (1 2 3)&&() +// // to -// (1 2 3)&&(1 2 3). +// +// (1 2 3)&&(1 2 3). // // The supplied changes are then applied to the incoming majority config, // resulting in a joint configuration that in terms of the Raft thesis[1] diff --git a/raft/doc.go b/raft/doc.go index d491352d441d..4febfe6084b8 100644 --- a/raft/doc.go +++ b/raft/doc.go @@ -25,46 +25,46 @@ A simple example application, _raftexample_, is also available to help illustrat how to use this package in practice: https://github.com/etcd-io/etcd/tree/main/contrib/raftexample -Usage +# Usage The primary object in raft is a Node. You either start a Node from scratch using raft.StartNode or start a Node from some initial state using raft.RestartNode. To start a node from scratch: - storage := raft.NewMemoryStorage() - c := &Config{ - ID: 0x01, - ElectionTick: 10, - HeartbeatTick: 1, - Storage: storage, - MaxSizePerMsg: 4096, - MaxInflightMsgs: 256, - } - n := raft.StartNode(c, []raft.Peer{{ID: 0x02}, {ID: 0x03}}) + storage := raft.NewMemoryStorage() + c := &Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + } + n := raft.StartNode(c, []raft.Peer{{ID: 0x02}, {ID: 0x03}}) To restart a node from previous state: - storage := raft.NewMemoryStorage() - - // recover the in-memory storage from persistent - // snapshot, state and entries. - storage.ApplySnapshot(snapshot) - storage.SetHardState(state) - storage.Append(entries) - - c := &Config{ - ID: 0x01, - ElectionTick: 10, - HeartbeatTick: 1, - Storage: storage, - MaxSizePerMsg: 4096, - MaxInflightMsgs: 256, - } + storage := raft.NewMemoryStorage() + + // recover the in-memory storage from persistent + // snapshot, state and entries. + storage.ApplySnapshot(snapshot) + storage.SetHardState(state) + storage.Append(entries) + + c := &Config{ + ID: 0x01, + ElectionTick: 10, + HeartbeatTick: 1, + Storage: storage, + MaxSizePerMsg: 4096, + MaxInflightMsgs: 256, + } - // restart raft without peer information. - // peer information is already included in the storage. - n := raft.RestartNode(c) + // restart raft without peer information. + // peer information is already included in the storage. + n := raft.RestartNode(c) Now that you are holding onto a Node you have a few responsibilities: @@ -120,29 +120,29 @@ represented by an abstract "tick". The total state machine handling loop will look something like this: - for { - select { - case <-s.Ticker: - n.Tick() - case rd := <-s.Node.Ready(): - saveToStorage(rd.State, rd.Entries, rd.Snapshot) - send(rd.Messages) - if !raft.IsEmptySnap(rd.Snapshot) { - processSnapshot(rd.Snapshot) - } - for _, entry := range rd.CommittedEntries { - process(entry) - if entry.Type == raftpb.EntryConfChange { - var cc raftpb.ConfChange - cc.Unmarshal(entry.Data) - s.Node.ApplyConfChange(cc) - } - } - s.Node.Advance() - case <-s.done: - return - } - } + for { + select { + case <-s.Ticker: + n.Tick() + case rd := <-s.Node.Ready(): + saveToStorage(rd.State, rd.Entries, rd.Snapshot) + send(rd.Messages) + if !raft.IsEmptySnap(rd.Snapshot) { + processSnapshot(rd.Snapshot) + } + for _, entry := range rd.CommittedEntries { + process(entry) + if entry.Type == raftpb.EntryConfChange { + var cc raftpb.ConfChange + cc.Unmarshal(entry.Data) + s.Node.ApplyConfChange(cc) + } + } + s.Node.Advance() + case <-s.done: + return + } + } To propose changes to the state machine from your node take your application data, serialize it into a byte slice and call: @@ -169,7 +169,7 @@ given ID MUST be used only once even if the old node has been removed. This means that for example IP addresses make poor node IDs since they may be reused. Node IDs must be non-zero. -Implementation notes +# Implementation notes This implementation is up to date with the final Raft thesis (https://github.com/ongardie/dissertation/blob/master/stanford.pdf), although our @@ -194,7 +194,7 @@ cannot be removed any more since the cluster cannot make progress. For this reason it is highly recommended to use three or more nodes in every cluster. -MessageType +# MessageType Package raft sends and receives message in Protocol Buffer format (defined in raftpb package). Each state (follower, candidate, leader) implements its @@ -295,6 +295,5 @@ stale log entries: that the follower that sent this 'MsgUnreachable' is not reachable, often indicating 'MsgApp' is lost. When follower's progress state is replicate, the leader sets it back to probe. - */ package raft diff --git a/raft/go.mod b/raft/go.mod index a559403e740a..c60344d80277 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -1,14 +1,20 @@ module go.etcd.io/etcd/raft/v3 -go 1.16 +go 1.23.0 + +toolchain go1.23.7 require ( - github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect - github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 + github.com/cockroachdb/datadriven v1.0.2 github.com/gogo/protobuf v1.3.2 - github.com/golang/protobuf v1.5.2 - github.com/pkg/errors v0.9.1 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.0-alpha.0 + github.com/golang/protobuf v1.5.4 + go.etcd.io/etcd/client/pkg/v3 v3.5.21 +) + +require ( + github.com/google/go-cmp v0.6.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) // Bad imports are sometimes causing attempts to pull that code. diff --git a/raft/go.sum b/raft/go.sum index 622e9f3eb18e..ec8b9fbfe3e9 100644 --- a/raft/go.sum +++ b/raft/go.sum @@ -1,142 +1,42 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19/go.mod h1:aMfIlz3TDBfB0BwTCKFU1XbEmj9zevr5S5LcBr85MXw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/raft/log_test.go b/raft/log_test.go index 0ad23f93ba10..bcb29e94afd2 100644 --- a/raft/log_test.go +++ b/raft/log_test.go @@ -146,12 +146,14 @@ func TestAppend(t *testing.T) { // TestLogMaybeAppend ensures: // If the given (index, term) matches with the existing log: -// 1. If an existing entry conflicts with a new one (same index -// but different terms), delete the existing entry and all that -// follow it -// 2.Append any new entries not already in the log +// 1. If an existing entry conflicts with a new one (same index +// but different terms), delete the existing entry and all that +// follow it +// 2. Append any new entries not already in the log +// // If the given (index, term) does not match with the existing log: -// return false +// +// return false func TestLogMaybeAppend(t *testing.T) { previousEnts := []pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}} lastindex := uint64(3) @@ -528,7 +530,7 @@ func TestStableToWithSnap(t *testing.T) { } } -//TestCompaction ensures that the number of log entries is correct after compactions. +// TestCompaction ensures that the number of log entries is correct after compactions. func TestCompaction(t *testing.T) { tests := []struct { lastIndex uint64 diff --git a/raft/node_test.go b/raft/node_test.go index e44d073e2686..cc6acae458b9 100644 --- a/raft/node_test.go +++ b/raft/node_test.go @@ -76,7 +76,7 @@ func TestNodeStep(t *testing.T) { } } -// Cancel and Stop should unblock Step() +// TestNodeStepUnblock should Cancel and Stop should unblock Step() func TestNodeStepUnblock(t *testing.T) { // a node without buffer to block step n := &node{ @@ -959,14 +959,14 @@ func (s *ignoreSizeHintMemStorage) Entries(lo, hi uint64, maxSize uint64) ([]raf // Storage's Entries size limitation is slightly more permissive than Raft's // internal one. The original bug was the following: // -// - node learns that index 11 (or 100, doesn't matter) is committed -// - nextEnts returns index 1..10 in CommittedEntries due to size limiting. However, -// index 10 already exceeds maxBytes, due to a user-provided impl of Entries. -// - Commit index gets bumped to 10 -// - the node persists the HardState, but crashes before applying the entries -// - upon restart, the storage returns the same entries, but `slice` takes a different code path -// (since it is now called with an upper bound of 10) and removes the last entry. -// - Raft emits a HardState with a regressing commit index. +// - node learns that index 11 (or 100, doesn't matter) is committed +// - nextEnts returns index 1..10 in CommittedEntries due to size limiting. However, +// index 10 already exceeds maxBytes, due to a user-provided impl of Entries. +// - Commit index gets bumped to 10 +// - the node persists the HardState, but crashes before applying the entries +// - upon restart, the storage returns the same entries, but `slice` takes a different code path +// (since it is now called with an upper bound of 10) and removes the last entry. +// - Raft emits a HardState with a regressing commit index. // // A simpler version of this test would have the storage return a lot less entries than dictated // by maxSize (for example, exactly one entry) after the restart, resulting in a larger regression. diff --git a/raft/raft_test.go b/raft/raft_test.go index f21670776b76..2579906b4adc 100644 --- a/raft/raft_test.go +++ b/raft/raft_test.go @@ -1207,8 +1207,8 @@ func TestPastElectionTimeout(t *testing.T) { } } -// ensure that the Step function ignores the message from old term and does not pass it to the -// actual stepX function. +// TestStepIgnoreOldTermMsg to ensure that the Step function ignores the message +// from old term and does not pass it to the actual stepX function. func TestStepIgnoreOldTermMsg(t *testing.T) { called := false fakeStep := func(r *raft, m pb.Message) error { @@ -1225,10 +1225,10 @@ func TestStepIgnoreOldTermMsg(t *testing.T) { } // TestHandleMsgApp ensures: -// 1. Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm. -// 2. If an existing entry conflicts with a new one (same index but different terms), -// delete the existing entry and all that follow it; append any new entries not already in the log. -// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry). +// 1. Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm. +// 2. If an existing entry conflicts with a new one (same index but different terms), +// delete the existing entry and all that follow it; append any new entries not already in the log. +// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry). func TestHandleMsgApp(t *testing.T) { tests := []struct { m pb.Message @@ -2479,7 +2479,7 @@ func TestLeaderAppResp(t *testing.T) { } } -// When the leader receives a heartbeat tick, it should +// TestBcastBeat is when the leader receives a heartbeat tick, it should // send a MsgHeartbeat with m.Index = 0, m.LogTerm=0 and empty entries. func TestBcastBeat(t *testing.T) { offset := uint64(1000) @@ -2539,7 +2539,7 @@ func TestBcastBeat(t *testing.T) { } } -// tests the output of the state machine when receiving MsgBeat +// TestRecvMsgBeat tests the output of the state machine when receiving MsgBeat func TestRecvMsgBeat(t *testing.T) { tests := []struct { state StateType @@ -2818,7 +2818,7 @@ func TestRestoreWithLearner(t *testing.T) { } } -/// Tests if outgoing voter can receive and apply snapshot correctly. +// TestRestoreWithVotersOutgoing tests if outgoing voter can receive and apply snapshot correctly. func TestRestoreWithVotersOutgoing(t *testing.T) { s := pb.Snapshot{ Metadata: pb.SnapshotMetadata{ @@ -4306,12 +4306,12 @@ func testConfChangeCheckBeforeCampaign(t *testing.T, v2 bool) { } } -// Tests if unapplied ConfChange is checked before campaign. +// TestConfChangeCheckBeforeCampaign tests if unapplied ConfChange is checked before campaign. func TestConfChangeCheckBeforeCampaign(t *testing.T) { testConfChangeCheckBeforeCampaign(t, false) } -// Tests if unapplied ConfChangeV2 is checked before campaign. +// TestConfChangeV2CheckBeforeCampaign tests if unapplied ConfChangeV2 is checked before campaign. func TestConfChangeV2CheckBeforeCampaign(t *testing.T) { testConfChangeCheckBeforeCampaign(t, true) } diff --git a/raft/rawnode_test.go b/raft/rawnode_test.go index 898b0f12c3e2..6c7c15347244 100644 --- a/raft/rawnode_test.go +++ b/raft/rawnode_test.go @@ -36,22 +36,22 @@ type rawNodeAdapter struct { var _ Node = (*rawNodeAdapter)(nil) -// Node specifies lead, which is pointless, can just be filled in. +// TransferLeadership is to test when node specifies lead, which is pointless, can just be filled in. func (a *rawNodeAdapter) TransferLeadership(ctx context.Context, lead, transferee uint64) { a.RawNode.TransferLeader(transferee) } -// Node has a goroutine, RawNode doesn't need this. +// Stop when node has a goroutine, RawNode doesn't need this. func (a *rawNodeAdapter) Stop() {} -// RawNode returns a *Status. +// Status returns RawNode's status as *Status. func (a *rawNodeAdapter) Status() Status { return a.RawNode.Status() } -// RawNode takes a Ready. It doesn't really have to do that I think? It can hold on +// Advance is when RawNode takes a Ready. It doesn't really have to do that I think? It can hold on // to it internally. But maybe that approach is frail. func (a *rawNodeAdapter) Advance() { a.RawNode.Advance(Ready{}) } -// RawNode returns a Ready, not a chan of one. +// Ready when RawNode returns a Ready, not a chan of one. func (a *rawNodeAdapter) Ready() <-chan Ready { return nil } // Node takes more contexts. Easy enough to fix. @@ -868,17 +868,17 @@ func TestRawNodeStatus(t *testing.T) { // TestNodeCommitPaginationAfterRestart. The anomaly here was even worse as the // Raft group would forget to apply entries: // -// - node learns that index 11 is committed -// - nextEnts returns index 1..10 in CommittedEntries (but index 10 already -// exceeds maxBytes), which isn't noticed internally by Raft -// - Commit index gets bumped to 10 -// - the node persists the HardState, but crashes before applying the entries -// - upon restart, the storage returns the same entries, but `slice` takes a -// different code path and removes the last entry. -// - Raft does not emit a HardState, but when the app calls Advance(), it bumps -// its internal applied index cursor to 10 (when it should be 9) -// - the next Ready asks the app to apply index 11 (omitting index 10), losing a -// write. +// - node learns that index 11 is committed +// - nextEnts returns index 1..10 in CommittedEntries (but index 10 already +// exceeds maxBytes), which isn't noticed internally by Raft +// - Commit index gets bumped to 10 +// - the node persists the HardState, but crashes before applying the entries +// - upon restart, the storage returns the same entries, but `slice` takes a +// different code path and removes the last entry. +// - Raft does not emit a HardState, but when the app calls Advance(), it bumps +// its internal applied index cursor to 10 (when it should be 9) +// - the next Ready asks the app to apply index 11 (omitting index 10), losing a +// write. func TestRawNodeCommitPaginationAfterRestart(t *testing.T) { s := &ignoreSizeHintMemStorage{ MemoryStorage: newTestMemoryStorage(withPeers(1)), diff --git a/scripts/build-binary b/scripts/build-binary index 6186424aa9ef..f9c6f55c8988 100755 --- a/scripts/build-binary +++ b/scripts/build-binary @@ -1,19 +1,17 @@ #!/usr/bin/env bash -set -e +set -euo pipefail source ./scripts/test_lib.sh -VER=$1 +VER=${1:-} REPOSITORY="${REPOSITORY:-git@github.com:etcd-io/etcd.git}" - -if [ -z "$1" ]; then +if [ -z "${VER}" ]; then echo "Usage: ${0} VERSION" >> /dev/stderr exit 255 fi -set -u function setup_env { local ver=${1} @@ -25,9 +23,7 @@ function setup_env { pushd "${proj}" >/dev/null run git fetch --all - git_assert_branch_in_sync || exit 2 run git checkout "${ver}" - git_assert_branch_in_sync || exit 2 popd >/dev/null } @@ -41,7 +37,7 @@ function package { srcdir="${ccdir}" fi local ext="" - if [ "${GOOS}" == "windows" ]; then + if [ "${GOOS:-}" == "windows" ]; then ext=".exe" fi for bin in etcd etcdctl etcdutl; do @@ -63,7 +59,7 @@ function main { cd release setup_env "${VER}" "${proj}" - tarcmd=tar + local tarcmd=tar if [[ $(go env GOOS) == "darwin" ]]; then echo "Please use linux machine for release builds." exit 1 @@ -79,11 +75,15 @@ function main { TARGET_ARCHS+=("s390x") fi + if [ ${GOOS} == "darwin" ]; then + TARGET_ARCHS+=("arm64") + fi + for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do export GOARCH=${TARGET_ARCH} pushd etcd >/dev/null - GO_LDFLAGS="-s" ./build.sh + GO_LDFLAGS="-s -w" ./build.sh popd >/dev/null TARGET="etcd-${VER}-${GOOS}-${GOARCH}" diff --git a/scripts/build-docker b/scripts/build-docker index bc5fd7a10c7f..ffdcfc86f81b 100755 --- a/scripts/build-docker +++ b/scripts/build-docker @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e +set -euo pipefail if [ "$#" -ne 1 ]; then echo "Usage: $0 VERSION" >&2 @@ -11,7 +11,7 @@ ARCH=$(go env GOARCH) VERSION="${1}-${ARCH}" DOCKERFILE="Dockerfile-release.${ARCH}" -if [ -z "${BINARYDIR}" ]; then +if [ -z "${BINARYDIR:-}" ]; then RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH) BINARYDIR="${RELEASE}" TARFILE="${RELEASE}.tar.gz" @@ -30,13 +30,15 @@ IMAGEDIR=${BUILDDIR}/image-docker mkdir -p "${IMAGEDIR}"/var/etcd mkdir -p "${IMAGEDIR}"/var/lib/etcd -cp "${BINARYDIR}"/etcd "${BINARYDIR}"/etcdctl "${IMAGEDIR}" +cp "${BINARYDIR}"/etcd "${BINARYDIR}"/etcdctl "${BINARYDIR}"/etcdutl "${IMAGEDIR}" cat ./"${DOCKERFILE}" > "${IMAGEDIR}"/Dockerfile -if [ -z "$TAG" ]; then - docker build -t "gcr.io/etcd-development/etcd:${VERSION}" "${IMAGEDIR}" - docker build -t "quay.io/coreos/etcd:${VERSION}" "${IMAGEDIR}" +if [ -z "${TAG:-}" ]; then + # Fix incorrect image "Architecture" using buildkit + # From https://stackoverflow.com/q/72144329/ + DOCKER_BUILDKIT=1 docker build -t "gcr.io/etcd-development/etcd:${VERSION}" "${IMAGEDIR}" + DOCKER_BUILDKIT=1 docker build -t "quay.io/coreos/etcd:${VERSION}" "${IMAGEDIR}" else docker build -t "${TAG}:${VERSION}" "${IMAGEDIR}" fi diff --git a/scripts/build-release.sh b/scripts/build-release.sh index 149be75f10e1..b1600406458d 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -3,11 +3,11 @@ # Build all release binaries and images to directory ./release. # Run from repository root. # -set -e +set -euo pipefail source ./scripts/test_lib.sh -VERSION=$1 +VERSION=${1:-} if [ -z "${VERSION}" ]; then echo "Usage: ${0} VERSION" >> /dev/stderr exit 255 diff --git a/scripts/install-marker.sh b/scripts/install-marker.sh index 467492666d1f..495b0ec74b26 100755 --- a/scripts/install-marker.sh +++ b/scripts/install-marker.sh @@ -10,7 +10,7 @@ if [ -z "$1" ]; then fi MARKER_URL=https://storage.googleapis.com/etcd/test-binaries/marker-v0.4.0-x86_64-unknown-linux-gnu -if [ ${ARCH} == "darwin" ]; then +if [ "${ARCH}" == "darwin" ]; then MARKER_URL=https://storage.googleapis.com/etcd/test-binaries/marker-v0.4.0-x86_64-apple-darwin fi diff --git a/scripts/release b/scripts/release index ec40bb8b5770..b7778ab4ec01 100755 --- a/scripts/release +++ b/scripts/release @@ -1,322 +1,8 @@ #!/usr/bin/env bash -set -o errexit -set -o nounset -set -o pipefail +set -euo pipefail -source ./scripts/test_lib.sh -source ./scripts/release_mod.sh +echo -e "\\e[91mDEPRECATED!!! Use scripts/release.sh script instead.\\e[0m\\n" +sleep 1 -DRY_RUN=${DRY_RUN:-true} - -# Following preparation steps help with the release process: - -# If you use password-protected gpg key, make sure the password is managed -# by agent: -# -# % gpg-connect-agent reloadagent /bye -# % gpg -s --default-key [git-email]@google.com -o /dev/null -s /dev/null -# -# Refresh your google credentials: -# % gcloud auth login -# or -# % gcloud auth activate-service-account --key-file=gcp-key-etcd-development.json -# -# Make sure gcloud-docker plugin is configured: -# % gcloud auth configure-docker - - -help() { - echo "$(basename "$0") [version]" - echo "Release etcd using the same approach as the etcd-release-runbook (https://goo.gl/Gxwysq)" - echo "" - echo "WARNING: This does not perform the 'Add API capabilities', 'Performance testing' " - echo " or 'Documentation' steps. These steps must be performed manually BEFORE running this tool." - echo "" - echo "WARNING: This script does not sign releases, publish releases to github or sent announcement" - echo " emails. These steps must be performed manually AFTER running this tool." - echo "" - echo " args:" - echo " version: version of etcd to release, e.g. 'v3.2.18'" - echo " flags:" - echo " --no-upload: skip gs://etcd binary artifact uploads." - echo " --no-docker-push: skip docker image pushes." - echo "" - echo "One can perform a (dry-run) test release from any (uncommitted) branch using:" - echo " DRY_RUN=true REPOSITORY=\`pwd\` BRANCH='local-branch-name' ./scripts/release 3.5.0-foobar.2" -} - -main() { - VERSION=$1 - if [[ ! "${VERSION}" =~ [0-9]+.[0-9]+.[0-9]+ ]]; then - log_error "Expected 'version' param of the form '..' but got '${VERSION}'" - exit 1 - fi - RELEASE_VERSION="v${VERSION}" - MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2) - BRANCH=${BRANCH:-"release-${MINOR_VERSION}"} - REPOSITORY=${REPOSITORY:-"git@github.com:etcd-io/etcd.git"} - - log_warning "DRY_RUN=${DRY_RUN}" - log_callout "RELEASE_VERSION=${RELEASE_VERSION}" - log_callout "MINOR_VERSION=${MINOR_VERSION}" - log_callout "BRANCH=${BRANCH}" - log_callout "REPOSITORY=${REPOSITORY}" - log_callout "" - - # Required to enable 'docker manifest ...' - export DOCKER_CLI_EXPERIMENTAL=enabled - - if ! command -v docker >/dev/null; then - log_error "cannot find docker" - exit 1 - fi - - # Expected umask for etcd release artifacts - umask 022 - - # Set up release directory. - local reldir="/tmp/etcd-release-${VERSION}" - log_callout "Preparing temporary directory: ${reldir}" - if [ ! -d "${reldir}/etcd" ]; then - mkdir -p "${reldir}" - cd "${reldir}" - run git clone "${REPOSITORY}" --branch "${BRANCH}" - fi - run cd "${reldir}/etcd" || exit 2 - # mark local directory as root for test_lib scripts executions - set_root_dir - - run git checkout "${BRANCH}" || exit 2 - run git pull origin - git_assert_branch_in_sync || exit 2 - - # If a release version tag already exists, use it. - local remote_tag_exists - remote_tag_exists=$(run git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}" || true) - - if [ "${remote_tag_exists}" -gt 0 ]; then - log_callout "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}" - git checkout -q "tags/${RELEASE_VERSION}" - fi - - # Check go version. - local go_version current_go_version - go_version="go$(run_go_tool "github.com/mikefarah/yq/v3" read .travis.yml "go[0]")" - current_go_version=$(go version | awk '{ print $3 }') - if [[ "${current_go_version}" != "${go_version}" ]]; then - log_error "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .travis.yml)." - exit 1 - fi - - # If the release tag does not already exist remotely, create it. - if [ "${remote_tag_exists}" -eq 0 ]; then - # Bump version/version.go to release version. - local source_version - source_version=$(grep -E "\s+Version\s*=" api/version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g") - if [[ "${source_version}" != "${VERSION}" ]]; then - source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2) - if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then - log_error "Wrong etcd minor version in api/version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting." - exit 1 - fi - log_callout "Updating modules definitions" - TARGET_VERSION="v${VERSION}" update_versions_cmd - - log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go" - sed -i "s/${source_version}/${VERSION}/g" api/version/version.go - fi - - - log_callout "Building etcd and checking --version output" - run ./build.sh - local etcd_version - etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }') - if [[ "${etcd_version}" != "${VERSION}" ]]; then - log_error "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting." - exit 1 - fi - - if [[ -n $(git status -s) ]]; then - log_callout "Committing mods & api/version/version.go update." - run git add api/version/version.go - run git add $(find -name go.mod ! -path './release/*'| xargs) - run git diff --staged | cat - run git commit -m "version: bump up to ${VERSION}" - run git diff --staged | cat - fi - - # Push the version change if it's not already been pushed. - if [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then - read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm - [[ "${confirm,,}" == "y" ]] || exit 1 - maybe_run git push - fi - - # Tag release. - if [ "$(git tag --list | grep -c "${RELEASE_VERSION}")" -gt 0 ]; then - log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists." - else - log_callout "Tagging release..." - REMOTE_REPO="origin" push_mod_tags_cmd - fi - - # Verify the latest commit has the version tag - local tag="$(git describe --exact-match HEAD)" - if [ "${tag}" != "${RELEASE_VERSION}" ]; then - log_error "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}" - exit 1 - fi - - # Verify the version tag is on the right branch - local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3) - if [ "${branch}" != "${BRANCH}" ]; then - log_error "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'" - exit 1 - fi - fi - - # Build release. - # TODO: check the release directory for all required build artifacts. - if [ -d release ]; then - log_warning "Skipping release build step. /release directory already exists." - else - log_callout "Building release..." - if ${DRY_RUN}; then - log_warning "In DRY_RUN mode we clone the current release directory (as there was no push)" - REPOSITORY=$(pwd) ./scripts/build-release.sh "${RELEASE_VERSION}" - else - REPOSITORY=${REPOSITORY} ./scripts/build-release.sh "${RELEASE_VERSION}" - fi - fi - - # Sanity checks. - "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep -q "etcd Version: ${VERSION}" || true - "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdctl" version | grep -q "etcdctl version: ${VERSION}" || true - - # Generate SHA256SUMS - log_callout "Generating sha256sums of release artifacts." - pushd ./release - ls . | grep -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS - popd - if [ -s ./release/SHA256SUMS ]; then - cat ./release/SHA256SUMS - else - log_error "sha256sums is not valid. Aborting." - exit 1 - fi - - # Upload artifacts. - if [ "${NO_UPLOAD}" == 1 ]; then - log_callout "Skipping artifact upload to gs://etcd. --no-upload flat is set." - else - read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " -r confirm - [[ "${confirm,,}" == "y" ]] || exit 1 - maybe_run gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/" - maybe_run gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/" - maybe_run gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/" - maybe_run gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/" - fi - - # Push images. - if [ "${NO_DOCKER_PUSH}" == 1 ]; then - log_callout "Skipping docker push. --no-docker-push flat is set." - else - read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " -r confirm - [[ "${confirm,,}" == "y" ]] || exit 1 - # shellcheck disable=SC2034 - for i in {1..5}; do - docker login quay.io && break - log_warning "login failed, retrying" - done - - for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do - log_callout "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}" - maybe_run docker push "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" - log_callout "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}" - maybe_run docker push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" - done - - log_callout "Creating manifest-list (multi-image)..." - - for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do - maybe_run docker manifest create --amend "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" - maybe_run docker manifest annotate "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}" - - maybe_run docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" - maybe_run docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}" - done - - log_callout "Pushing container manifest list to quay.io ${RELEASE_VERSION}" - maybe_run docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}" - - log_callout "Pushing container manifest list to gcr.io ${RELEASE_VERSION}" - maybe_run docker manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" - - log_callout "Setting permissions using gsutil..." - maybe_run gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com - fi - - ### Release validation - mkdir -p downloads - - # Check image versions - for IMAGE in "quay.io/coreos/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"; do - local image_version=$(dry_run docker run --rm "${IMAGE}" etcd --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]') - if [ "${image_version}" != "${VERSION}" ]; then - log_error "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}" - exit 1 - fi - done - - # Check gsutil binary versions - local BINARY_TGZ="etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64.tar.gz" - gsutil cp "gs://etcd/${RELEASE_VERSION}/${BINARY_TGZ}" downloads - tar -zx -C downloads -f "downloads/${BINARY_TGZ}" - local binary_version=$("./downloads/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]') - if [ "${binary_version}" != "${VERSION}" ]; then - log_error "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}" - exit 1 - fi - - # TODO: signing process - log_warning "" - log_warning "WARNING: The release has not been signed and published to github. This must be done manually." - log_warning "" - log_success "Success." - exit 0 -} - -POSITIONAL=() -NO_UPLOAD=0 -NO_DOCKER_PUSH=0 - -while test $# -gt 0; do - case "$1" in - -h|--help) - shift - help - exit 0 - ;; - --no-upload) - NO_UPLOAD=1 - shift - ;; - --no-docker-push) - NO_DOCKER_PUSH=1 - shift - ;; - *) - POSITIONAL+=("$1") # save it in an array for later - shift # past argument - ;; - esac -done -set -- "${POSITIONAL[@]}" # restore positional parameters - -if [[ ! $# -eq 1 ]]; then - help - exit 1 -fi - -main "$1" +source ./scripts/release.sh diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 000000000000..635fbfc1248a --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,451 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +source ./scripts/test_lib.sh +source ./scripts/release_mod.sh + +DRY_RUN=${DRY_RUN:-true} + +# Following preparation steps help with the release process: + +# If you use password-protected gpg key, make sure the password is managed +# by agent: +# +# % gpg-connect-agent reloadagent /bye +# % gpg -s --default-key [git-email]@google.com -o /dev/null -s /dev/null +# +# Refresh your google credentials: +# % gcloud auth login +# or +# % gcloud auth activate-service-account --key-file=gcp-key-etcd-development.json +# +# Make sure gcloud-docker plugin is configured: +# % gcloud auth configure-docker + + +help() { + echo "$(basename "$0") [version]" + echo "Release etcd using the same approach as the etcd-release-runbook (https://goo.gl/Gxwysq)" + echo "" + echo "WARNING: This does not perform the 'Add API capabilities', 'Performance testing' " + echo " or 'Documentation' steps. These steps must be performed manually BEFORE running this tool." + echo "" + echo "WARNING: This script does not send announcement emails. This step must be performed manually AFTER running this tool." + echo "" + echo " args:" + echo " version: version of etcd to release, e.g. 'v3.2.18'" + echo " flags:" + echo " --in-place: build binaries using current branch." + echo " --no-docker-push: skip docker image pushes." + echo " --no-gh-release: skip creating the GitHub release using gh." + echo " --no-upload: skip gs://etcd binary artifact uploads." + echo "" + echo "One can perform a (dry-run) test release from any (uncommitted) branch using:" + echo " DRY_RUN=true REPOSITORY=\`pwd\` BRANCH='local-branch-name' ./scripts/release 3.5.0-foobar.2" +} + +main() { + VERSION=$1 + if [[ ! "${VERSION}" =~ [0-9]+.[0-9]+.[0-9]+ ]]; then + log_error "Expected 'version' param of the form '..' but got '${VERSION}'" + exit 1 + fi + RELEASE_VERSION="v${VERSION}" + MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2) + + if [ "${IN_PLACE}" == 1 ]; then + # Trigger release in current branch + REPOSITORY=$(pwd) + BRANCH=$(git rev-parse --abbrev-ref HEAD) + else + REPOSITORY=${REPOSITORY:-"git@github.com:etcd-io/etcd.git"} + BRANCH=${BRANCH:-"release-${MINOR_VERSION}"} + fi + + log_warning "DRY_RUN=${DRY_RUN}" + log_callout "RELEASE_VERSION=${RELEASE_VERSION}" + log_callout "MINOR_VERSION=${MINOR_VERSION}" + log_callout "BRANCH=${BRANCH}" + log_callout "REPOSITORY=${REPOSITORY}" + log_callout "" + + # Required to enable 'docker manifest ...' + export DOCKER_CLI_EXPERIMENTAL=enabled + + if ! command -v docker >/dev/null; then + log_error "cannot find docker" + exit 1 + fi + + # Expected umask for etcd release artifacts + umask 022 + + # Set up release directory. + local reldir="/tmp/etcd-release-${VERSION}" + log_callout "Preparing temporary directory: ${reldir}" + if [ "${IN_PLACE}" == 0 ]; then + if [ ! -d "${reldir}/etcd" ]; then + mkdir -p "${reldir}" + cd "${reldir}" + run git clone "${REPOSITORY}" --branch "${BRANCH}" --depth 1 + fi + run cd "${reldir}/etcd" || exit 2 + run git checkout "${BRANCH}" || exit 2 + run git pull origin + + git_assert_branch_in_sync || exit 2 + fi + + # mark local directory as root for test_lib scripts executions + set_root_dir + + # If a release version tag already exists, use it. + local remote_tag_exists + remote_tag_exists=$(run git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}" || true) + + if [ "${remote_tag_exists}" -gt 0 ]; then + log_callout "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}" + git checkout -q "tags/${RELEASE_VERSION}" + fi + + # Check go version. + log_callout "Check go version" + local go_version current_go_version + go_version="go$(cat .go-version)" + current_go_version=$(go version | awk '{ print $3 }') + if [[ "${current_go_version}" != "${go_version}" ]]; then + log_error "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .go-version)." + exit 1 + fi + + if [ "${NO_GH_RELEASE}" == 1 ]; then + log_callout "Skipping gh verification, --no-gh-release is set" + else + # Check that gh is installed and logged in. + log_callout "Check gh installation" + if ! command -v gh >/dev/null; then + log_error "Cannot find gh. Please follow the installation instructions at https://github.com/cli/cli#installation" + exit 1 + fi + if ! gh auth status &>/dev/null; then + log_error "GitHub authentication failed for gh. Please run gh auth login." + exit 1 + fi + fi + + # If the release tag does not already exist remotely, create it. + log_callout "Create tag if not present" + if [ "${remote_tag_exists}" -eq 0 ]; then + # Bump version/version.go to release version. + local source_version + source_version=$(grep -E "\s+Version\s*=" api/version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g") + if [[ "${source_version}" != "${VERSION}" ]]; then + source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2) + if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then + log_error "Wrong etcd minor version in api/version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting." + exit 1 + fi + log_callout "Updating modules definitions" + TARGET_VERSION="v${VERSION}" update_versions_cmd + + log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go" + sed -i "s/${source_version}/${VERSION}/g" api/version/version.go + fi + + + log_callout "Building etcd and checking --version output" + run ./build.sh + local etcd_version + etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }') + if [[ "${etcd_version}" != "${VERSION}" ]]; then + log_error "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting." + exit 1 + fi + + if [[ -n $(git status -s) ]]; then + log_callout "Committing mods & api/version/version.go update." + run git add api/version/version.go + # shellcheck disable=SC2038,SC2046,SC2185 + run git add $(find -name go.mod ! -path './release/*'| xargs) + run git diff --staged | cat + run git commit --signoff --message "version: bump up to ${VERSION}" + run git diff --staged | cat + fi + + # Push the version change if it's not already been pushed. + if [ "${DRY_RUN}" != "true" ] && [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then + read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm + [[ "${confirm,,}" == "y" ]] || exit 1 + maybe_run git push + fi + + # Tag release. + if [ "$(git tag --list | grep -c "${RELEASE_VERSION}")" -gt 0 ]; then + log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists." + else + log_callout "Tagging release..." + REMOTE_REPO="origin" push_mod_tags_cmd + fi + + if [ "${IN_PLACE}" == 0 ]; then + # Tried with `local branch=$(git branch -a --contains tags/"${RELEASE_VERSION}")` + # so as to work with both current branch and main/release-3.X. + # But got error below on current branch mode, + # Error: Git tag v3.6.99 should be on branch '* (HEAD detached at pull/14860/merge)' but is on '* (HEAD detached from pull/14860/merge)' + # + # Verify the version tag is on the right branch + # shellcheck disable=SC2155 + local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3) + if [ "${branch}" != "${BRANCH}" ]; then + log_error "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'" + exit 1 + fi + fi + fi + + log_callout "Verify the latest commit has the version tag" + # Verify the latest commit has the version tag + # shellcheck disable=SC2155 + local tag="$(git describe --exact-match HEAD)" + if [ "${tag}" != "${RELEASE_VERSION}" ]; then + log_error "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}" + exit 1 + fi + + log_callout "Verify the work space is clean" + # Verify the clean working tree + # shellcheck disable=SC2155 + local diff="$(git diff HEAD --stat)" + if [[ "${diff}" != '' ]]; then + log_error "Error: Expected clean working tree, but 'git diff --stat' reported: ${diff}" + exit 1 + fi + + # Build release. + # TODO: check the release directory for all required build artifacts. + if [ -d release ]; then + log_warning "Skipping release build step. /release directory already exists." + else + log_callout "Building release..." + REPOSITORY=$(pwd) ./scripts/build-release.sh "${RELEASE_VERSION}" + fi + + # Sanity checks. + "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep -q "etcd Version: ${VERSION}" || true + "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdctl" version | grep -q "etcdctl version: ${VERSION}" || true + "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdutl" version | grep -q "etcdutl version: ${VERSION}" || true + + # Generate SHA256SUMS + log_callout "Generating sha256sums of release artifacts." + pushd ./release + # shellcheck disable=SC2010 + ls . | grep -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS + popd + if [ -s ./release/SHA256SUMS ]; then + cat ./release/SHA256SUMS + else + log_error "sha256sums is not valid. Aborting." + exit 1 + fi + + # Upload artifacts. + if [ "${DRY_RUN}" == "true" ] || [ "${NO_UPLOAD}" == 1 ]; then + log_callout "Skipping artifact upload to gs://etcd. --no-upload flat is set." + else + read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " -r confirm + [[ "${confirm,,}" == "y" ]] || exit 1 + maybe_run gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/" + maybe_run gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/" + maybe_run gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/" + maybe_run gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/" + fi + + # Push images. + if [ "${DRY_RUN}" == "true" ] || [ "${NO_DOCKER_PUSH}" == 1 ]; then + log_callout "Skipping docker push. --no-docker-push flat is set." + else + read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " -r confirm + [[ "${confirm,,}" == "y" ]] || exit 1 + # shellcheck disable=SC2034 + for i in {1..5}; do + docker login quay.io && break + log_warning "login failed, retrying" + done + + for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do + log_callout "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}" + maybe_run docker push "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" + log_callout "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}" + maybe_run docker push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" + done + + log_callout "Creating manifest-list (multi-image)..." + + for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do + maybe_run docker manifest create --amend "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" + maybe_run docker manifest annotate "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}" + + maybe_run docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" + maybe_run docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}" + done + + log_callout "Pushing container manifest list to quay.io ${RELEASE_VERSION}" + maybe_run docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}" + + log_callout "Pushing container manifest list to gcr.io ${RELEASE_VERSION}" + maybe_run docker manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" + fi + + ### Release validation + mkdir -p downloads + + # Check image versions + for IMAGE in "quay.io/coreos/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"; do + if [ "${DRY_RUN}" == "true" ] || [ "${NO_DOCKER_PUSH}" == 1 ]; then + IMAGE="${IMAGE}-amd64" + fi + # shellcheck disable=SC2155 + local image_version=$(docker run --rm "${IMAGE}" etcd --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]') + if [ "${image_version}" != "${VERSION}" ]; then + log_error "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}" + exit 1 + fi + done + + # Check gsutil binary versions + # shellcheck disable=SC2155 + local BINARY_TGZ="etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64.tar.gz" + if [ "${DRY_RUN}" == "true" ] || [ "${NO_UPLOAD}" == 1 ]; then + cp "./release/${BINARY_TGZ}" downloads + else + gsutil cp "gs://etcd/${RELEASE_VERSION}/${BINARY_TGZ}" downloads + fi + tar -zx -C downloads -f "downloads/${BINARY_TGZ}" + # shellcheck disable=SC2155 + local binary_version=$("./downloads/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]') + if [ "${binary_version}" != "${VERSION}" ]; then + log_error "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}" + exit 1 + fi + + if [ "${DRY_RUN}" == "true" ] || [ "${NO_GH_RELEASE}" == 1 ]; then + log_warning "" + log_warning "WARNING: Skipping creating GitHub release, --no-gh-release is set." + log_warning "WARNING: If not running on DRY_MODE, please do the GitHub release manually." + log_warning "" + else + local gh_repo + local release_notes_temp_file + local release_url + local gh_release_args=() + + # For the main branch (v3.6), we should mark the release as a prerelease. + # The release-3.5 (v3.5) branch, should be marked as latest. And release-3.4 (v3.4) + # should be left without any additional mark (therefore, it doesn't need a special argument). + if [ "${BRANCH}" = "main" ]; then + gh_release_args=(--prerelease) + elif [ "${BRANCH}" = "release-3.5" ]; then + gh_release_args=(--latest) + fi + + if [ "${REPOSITORY}" = "$(pwd)" ]; then + gh_repo=$(git remote get-url origin) + else + gh_repo="${REPOSITORY}" + fi + + gh_repo=$(echo "${gh_repo}" | sed 's/^[^@]\+@//' | sed 's/https\?:\/\///' | sed 's/\.git$//' | tr ':' '/') + log_callout "Creating GitHub release for ${RELEASE_VERSION} on ${gh_repo}" + + release_notes_temp_file=$(mktemp) + + local release_version=${RELEASE_VERSION#v} # Remove the v prefix from the release version (i.e., v3.6.1 -> 3.6.1) + local release_version_major_minor=${release_version%.*} # Remove the patch from the version (i.e., 3.6) + local release_version_major=${release_version_major_minor%.*} # Extract the major (i.e., 3) + local release_version_minor=${release_version_major_minor/*./} # Extract the minor (i.e., 6) + + # Disable sellcheck SC2016, the single quoted syntax for sed is intentional. + # shellcheck disable=SC2016 + sed 's/${RELEASE_VERSION}/'"${RELEASE_VERSION}"'/g' ./scripts/release_notes.tpl.txt | + sed 's/${RELEASE_VERSION_MAJOR_MINOR}/'"${release_version_major_minor}"'/g' | + sed 's/${RELEASE_VERSION_MAJOR}/'"${release_version_major}"'/g' | + sed 's/${RELEASE_VERSION_MINOR}/'"${release_version_minor}"'/g' > "${release_notes_temp_file}" + + if ! gh --repo "${gh_repo}" release view "${RELEASE_VERSION}" &>/dev/null; then + maybe_run gh release create "${RELEASE_VERSION}" \ + --repo "${gh_repo}" \ + --draft \ + --title "${RELEASE_VERSION}" \ + --notes-file "${release_notes_temp_file}" \ + "${gh_release_args[@]}" + fi + + # Upload files one by one, as gh doesn't support passing globs as input. + maybe_run find ./release '(' -name '*.tar.gz' -o -name '*.zip' ')' -exec \ + gh --repo "${gh_repo}" release upload "${RELEASE_VERSION}" {} --clobber \; + maybe_run gh --repo "${gh_repo}" release upload "${RELEASE_VERSION}" ./release/SHA256SUMS --clobber + + release_url=$(gh --repo "${gh_repo}" release view "${RELEASE_VERSION}" --json url --jq '.url') + + log_warning "" + log_warning "WARNING: The GitHub release for ${RELEASE_VERSION} has been created as a draft, please go to ${release_url} and release it." + log_warning "" + fi + + log_success "Success." + exit 0 +} + +POSITIONAL=() +NO_UPLOAD=0 +NO_DOCKER_PUSH=0 +IN_PLACE=0 +NO_GH_RELEASE=0 + +while test $# -gt 0; do + case "$1" in + -h|--help) + shift + help + exit 0 + ;; + --in-place) + IN_PLACE=1 + shift + ;; + --no-upload) + NO_UPLOAD=1 + shift + ;; + --no-docker-push) + NO_DOCKER_PUSH=1 + shift + ;; + --no-gh-release) + NO_GH_RELEASE=1 + shift + ;; + *) + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; + esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +if [[ ! $# -eq 1 ]]; then + help + exit 1 +fi + +# Note that we shouldn't upload artifacts in --in-place mode, so it +# must be called with DRY_RUN=true +if [ "${DRY_RUN}" != "true" ] && [ "${IN_PLACE}" == 1 ]; then + log_error "--in-place should only be called with DRY_RUN=true" + exit 1 +fi + +main "$1" diff --git a/scripts/release_notes.tpl.txt b/scripts/release_notes.tpl.txt new file mode 100644 index 000000000000..67701d9886e6 --- /dev/null +++ b/scripts/release_notes.tpl.txt @@ -0,0 +1,91 @@ +Please check out [CHANGELOG](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-${RELEASE_VERSION_MAJOR_MINOR}.md) for a full list of changes. And make sure to read [upgrade guide](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/upgrades/upgrade_${RELEASE_VERSION_MAJOR}_${RELEASE_VERSION_MINOR}/) before upgrading etcd (there may be breaking changes). + +For installation guides, please check out [play.etcd.io](http://play.etcd.io) and [operating etcd](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/op-guide/). Latest support status for common architectures and operating systems can be found at [supported platforms](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/op-guide/supported-platform/). + +###### Linux + +```sh +ETCD_VER=${RELEASE_VERSION} + +# choose either URL +GOOGLE_URL=https://storage.googleapis.com/etcd +GITHUB_URL=https://github.com/etcd-io/etcd/releases/download +DOWNLOAD_URL=${GOOGLE_URL} + +rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz +rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test + +curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz +tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 +rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz + +/tmp/etcd-download-test/etcd --version +/tmp/etcd-download-test/etcdctl version +/tmp/etcd-download-test/etcdutl version + +# start a local etcd server +/tmp/etcd-download-test/etcd + +# write,read to etcd +/tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 put foo bar +/tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 get foo +``` + +###### macOS (Darwin) + +```sh +ETCD_VER=${RELEASE_VERSION} + +# choose either URL +GOOGLE_URL=https://storage.googleapis.com/etcd +GITHUB_URL=https://github.com/etcd-io/etcd/releases/download +DOWNLOAD_URL=${GOOGLE_URL} + +rm -f /tmp/etcd-${ETCD_VER}-darwin-amd64.zip +rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test + +curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-darwin-amd64.zip -o /tmp/etcd-${ETCD_VER}-darwin-amd64.zip +unzip /tmp/etcd-${ETCD_VER}-darwin-amd64.zip -d /tmp && rm -f /tmp/etcd-${ETCD_VER}-darwin-amd64.zip +mv /tmp/etcd-${ETCD_VER}-darwin-amd64/* /tmp/etcd-download-test && rm -rf mv /tmp/etcd-${ETCD_VER}-darwin-amd64 + +/tmp/etcd-download-test/etcd --version +/tmp/etcd-download-test/etcdctl version +/tmp/etcd-download-test/etcdutl version +``` + +###### Docker + +etcd uses [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd) as a primary container registry, and [`quay.io/coreos/etcd`](https://quay.io/coreos/etcd) as secondary. + +```sh +ETCD_VER=${RELEASE_VERSION} + +rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \ + docker rmi gcr.io/etcd-development/etcd:${ETCD_VER} || true && \ + docker run \ + -p 2379:2379 \ + -p 2380:2380 \ + --mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \ + --name etcd-gcr-${ETCD_VER} \ + gcr.io/etcd-development/etcd:${ETCD_VER} \ + /usr/local/bin/etcd \ + --name s1 \ + --data-dir /etcd-data \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://0.0.0.0:2379 \ + --listen-peer-urls http://0.0.0.0:2380 \ + --initial-advertise-peer-urls http://0.0.0.0:2380 \ + --initial-cluster s1=http://0.0.0.0:2380 \ + --initial-cluster-token tkn \ + --initial-cluster-state new \ + --log-level info \ + --logger zap \ + --log-outputs stderr + +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcd --version +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl version +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdutl version +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl endpoint health +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl put foo bar +docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl get foo +``` diff --git a/scripts/sync_go_toolchain_directive.sh b/scripts/sync_go_toolchain_directive.sh new file mode 100755 index 000000000000..643138e1d0a9 --- /dev/null +++ b/scripts/sync_go_toolchain_directive.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script looks at the version present in the .go-version file and treats +# that to be the value of the toolchain directive that go should use. It then +# updates the toolchain directives of all go.mod files to reflect this version. +# +# We do this to ensure that .go-version acts as the source of truth for go versions. + +set -euo pipefail + +source ./scripts/test_lib.sh + +TARGET_GO_VERSION="${TARGET_GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" +find . -name 'go.mod' -exec go mod edit -toolchain=go"${TARGET_GO_VERSION}" {} \; diff --git a/scripts/test_images.sh b/scripts/test_images.sh new file mode 100755 index 000000000000..00b23ac18931 --- /dev/null +++ b/scripts/test_images.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +source ./scripts/test_lib.sh + +function startContainer { + # run docker in the background + docker run -d --rm --name "${RUN_NAME}" "${IMAGE}" + + # wait for etcd daemon to bootstrap + sleep 5 +} + +function runVersionCheck { + Out=$(docker run --rm "${IMAGE}" "${@}") + foundVersion=$(echo "$Out" | head -1 | rev | cut -d" " -f 1 | rev ) + if [[ "${foundVersion}" != "${VERSION}" ]]; then + echo "error: Invalid Version. Got $foundVersion, expected $VERSION. Error: $Out" + exit 1 + fi +} + +# Can't proceed without docker +if ! command -v docker >/dev/null; then + log_error "cannot find docker" + exit 1 +fi + +# You can't run darwin binaries in linux containers +if [[ $(go env GOOS) == "darwin" ]]; then + echo "Please use linux machine for release builds." + exit 1 +fi + +# Pick defaults based on release workflow +ARCH=$(go env GOARCH) +REPOSITARY=${REPOSITARY:-"gcr.io/etcd-development/etcd"} +if [ -n "$VERSION" ]; then + # Expected Format: v3.6.99-amd64 + TAG=v"${VERSION}"-"${ARCH}" +else + echo "Terminating test, VERSION not supplied" + exit 1 +fi +IMAGE=${IMAGE:-"${REPOSITARY}:${TAG}"} + +# ETCD related values +RUN_NAME="test_etcd" +KEY="foo" +VALUE="bar" + +if [[ "$(docker images -q "${IMAGE}" 2> /dev/null)" == "" ]]; then + echo "${IMAGE} not present locally" + exit 1 +fi + +# Version check +runVersionCheck "/usr/local/bin/etcd" "--version" +runVersionCheck "/usr/local/bin/etcdctl" "version" +runVersionCheck "/usr/local/bin/etcdutl" "version" + +startContainer +# stop container +trap 'docker stop "${RUN_NAME}"' EXIT + + +# Put/Get check +PUT=$(docker exec "${RUN_NAME}" /usr/local/bin/etcdctl put "${KEY}" "${VALUE}") +if [ "${PUT}" != "OK" ]; then + echo "Problem with Putting in etcd" + exit 1 +fi + +GET=$(docker exec "${RUN_NAME}" /usr/local/bin/etcdctl get "$KEY" --print-value-only) +if [ "${GET}" != "${VALUE}" ]; then + echo "Problem with getting foo bar in etcd. Got ${GET}" + exit 1 +fi + +echo "Succesfully tested etcd local image ${TAG}" + +for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do + ARCH_TAG=v"${VERSION}"-"${TARGET_ARCH}" + IMG_ARCH=$(docker inspect --format '{{.Architecture}}' "${REPOSITARY}:${ARCH_TAG}") + if [ "${IMG_ARCH}" != "$TARGET_ARCH" ];then + echo "Incorrect docker image architecture" + exit 1 + fi + echo "Correct Architecture ${ARCH_TAG}" +done diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh index 9053f9ce8c71..586516703842 100644 --- a/scripts/test_lib.sh +++ b/scripts/test_lib.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + ROOT_MODULE="go.etcd.io/etcd" if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then @@ -71,7 +73,8 @@ function relativePath { local commonPart=$source local result="" - while [[ "${target#$commonPart}" == "${target}" ]]; do + # Refer to https://www.shellcheck.net/wiki/SC2295 + while [[ "${target#"$commonPart"}" == "${target}" ]]; do # no match, means that candidate common part is not correct # go up one level (reduce common part) commonPart="$(dirname "$commonPart")" @@ -90,7 +93,8 @@ function relativePath { # since we now have identified the common part, # compute the non-common part - local forwardPart="${target#$commonPart}" + # Refer to https://www.shellcheck.net/wiki/SC2295 + local forwardPart="${target#"$commonPart"}" # and now stick all parts together if [[ -n $result ]] && [[ -n $forwardPart ]]; then @@ -105,10 +109,10 @@ function relativePath { #### Discovery of files/packages within a go module ##### -# go_srcs_in_module [package] +# go_srcs_in_module # returns list of all not-generated go sources in the current (dir) module. function go_srcs_in_module { - go fmt -n "$1" | grep -Eo "([^ ]*)$" | grep -vE "(\\_test.go|\\.pb\\.go|\\.pb\\.gw.go)" + go list -f "{{with \$c:=.}}{{range \$f:=\$c.GoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{range \$f:=\$c.TestGoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{range \$f:=\$c.XTestGoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{end}}" ./... | grep -vE "(\\.pb\\.go|\\.pb\\.gw.go)" } # pkgs_in_module [optional:package_pattern] @@ -169,7 +173,7 @@ function module_dirs() { # maybe_run [cmd...] runs given command depending on the DRY_RUN flag. function maybe_run() { - if ${DRY_RUN}; then + if ${DRY_RUN:-}; then log_warning -e "# DRY_RUN:\\n % ${*}" else run "${@}" @@ -241,7 +245,7 @@ function go_test { local goTestFlags="" local goTestEnv="" - if [ "${VERBOSE}" == "1" ]; then + if [ "${VERBOSE:-}" == "1" ]; then goTestFlags="-v" fi @@ -301,23 +305,27 @@ function tool_exists { fi } -# Ensure gobin is available, as it runs majority of the tools -if ! command -v "gobin" >/dev/null; then - run env GO111MODULE=off go get github.com/myitcv/gobin || exit 1 -fi - # tool_get_bin [tool] - returns absolute path to a tool binary (or returns error) function tool_get_bin { - tool_exists "gobin" "GO111MODULE=off go get github.com/myitcv/gobin" || return 2 - local tool="$1" + local pkg_part="$1" if [[ "$tool" == *"@"* ]]; then + pkg_part=$(echo "${tool}" | cut -d'@' -f1) # shellcheck disable=SC2086 - run gobin ${GOBINARGS:-} -p "${tool}" || return 2 + run go install ${GOBINARGS:-} "${tool}" || return 2 else # shellcheck disable=SC2086 - run_for_module ./tools/mod run gobin ${GOBINARGS:-} -p -m --mod=readonly "${tool}" || return 2 + run_for_module ./tools/mod run go install ${GOBINARGS:-} "${tool}" || return 2 fi + + # remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3". + local cmd_base_name + cmd_base_name=$(basename "${pkg_part}") + if [[ ${cmd_base_name} =~ ^v[0-9]*$ ]]; then + pkg_part=$(dirname "${pkg_part}") + fi + + run_for_module ./tools/mod go list -f '{{.Target}}' "${pkg_part}" } # tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg. @@ -354,14 +362,16 @@ function assert_no_git_modifications { # - no differencing commits in relation to the origin/$branch function git_assert_branch_in_sync { local branch - branch=$(run git rev-parse --abbrev-ref HEAD) # TODO: When git 2.22 popular, change to: # branch=$(git branch --show-current) + branch=$(run git rev-parse --abbrev-ref HEAD) + log_callout "Verify the current branch '${branch}' is clean" if [[ $(run git status --porcelain --untracked-files=no) ]]; then log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes" log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard)" return 2 fi + log_callout "Verify the current branch '${branch}' is in sync with the 'origin/${branch}'" if [ -n "${branch}" ]; then ref_local=$(run git rev-parse "${branch}") ref_origin=$(run git rev-parse "origin/${branch}") @@ -374,3 +384,27 @@ function git_assert_branch_in_sync { log_warning "Cannot verify consistency with the origin, as git is on detached branch." fi } + +# The version present in the .go-verion is the default version that test and build scripts will use. +# However, it is possible to control the version that should be used with the help of env vars: +# - FORCE_HOST_GO: if set to a non-empty value, use the version of go installed in system's $PATH. +# - GO_VERSION: desired version of go to be used, might differ from what is present in .go-version. +# If empty, the value defaults to the version in .go-version. +function determine_go_version { + # Borrowing from how Kubernetes does this: + # https://github.com/kubernetes/kubernetes/blob/17854f0e0a153b06f9d0db096e2cd8ab2fa89c11/hack/lib/golang.sh#L510-L520 + # + # default GO_VERSION to content of .go-version + GO_VERSION="${GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" + if [ "${GOTOOLCHAIN:-auto}" != 'auto' ]; then + # no-op, just respect GOTOOLCHAIN + : + elif [ -n "${FORCE_HOST_GO:-}" ]; then + export GOTOOLCHAIN='local' + else + GOTOOLCHAIN="go${GO_VERSION}" + export GOTOOLCHAIN + fi +} + +determine_go_version diff --git a/scripts/verify_go_versions.sh b/scripts/verify_go_versions.sh new file mode 100755 index 000000000000..11d34f92a515 --- /dev/null +++ b/scripts/verify_go_versions.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# This script verifies that the value of the toolchain directive in the +# go.mod files always match that of the .go-version file to ensure that +# we accidentally don't test and release with differing versions of Go. + +set -euo pipefail + +source ./scripts/test_lib.sh + +target_go_version="${target_go_version:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" +log_info "expected go toolchain directive: go${target_go_version}" +log_info + +toolchain_out_of_sync="false" +go_line_violation="false" + +# verify_go_versions takes a go.mod filepath as an argument +# and checks if: +# (1) go directive <= version in .go-version +# (2) toolchain directive == version in .go-version +function verify_go_versions() { + # shellcheck disable=SC2086 + toolchain_version="$(go mod edit -json $1 | jq -r .Toolchain)" + # shellcheck disable=SC2086 + go_line_version="$(go mod edit -json $1 | jq -r .Go)" + if [[ "go${target_go_version}" != "${toolchain_version}" ]]; then + log_error "go toolchain directive out of sync for $1, got: ${toolchain_version}" + toolchain_out_of_sync="true" + fi + if ! printf '%s\n' "${go_line_version}" "${target_go_version}" | sort --check=silent --version-sort; then + log_error "go directive in $1 is greater than maximum allowed: go${target_go_version}" + go_line_violation="true" + fi +} + +while read -r mod; do + verify_go_versions "${mod}"; +done < <(find . -name 'go.mod') + +if [[ "${toolchain_out_of_sync}" == "true" ]]; then + log_error + log_error "Please run scripts/sync_go_toolchain_directive.sh or update .go-version to rectify this error" +fi + +if [[ "${go_line_violation}" == "true" ]]; then + log_error + log_error "Please update .go-version to rectify this error, any go directive should be <= .go-version" +fi + +if [[ "${go_line_violation}" == "true" ]] || [[ "${toolchain_out_of_sync}" == "true" ]]; then + exit 1 +fi diff --git a/server/auth/jwt.go b/server/auth/jwt.go index d286f92c2ec7..a797794494bd 100644 --- a/server/auth/jwt.go +++ b/server/auth/jwt.go @@ -21,7 +21,7 @@ import ( "errors" "time" - jwt "github.com/form3tech-oss/jwt-go" + "github.com/golang-jwt/jwt/v4" "go.uber.org/zap" ) @@ -42,7 +42,7 @@ func (t *tokenJWT) info(ctx context.Context, token string, rev uint64) (*AuthInf // rev isn't used in JWT, it is only used in simple token var ( username string - revision uint64 + revision float64 ) parsed, err := jwt.Parse(token, func(token *jwt.Token) (interface{}, error) { @@ -74,10 +74,19 @@ func (t *tokenJWT) info(ctx context.Context, token string, rev uint64) (*AuthInf return nil, false } - username = claims["username"].(string) - revision = uint64(claims["revision"].(float64)) + username, ok = claims["username"].(string) + if !ok { + t.lg.Warn("failed to obtain user claims from jwt token") + return nil, false + } + + revision, ok = claims["revision"].(float64) + if !ok { + t.lg.Warn("failed to obtain revision claims from jwt token") + return nil, false + } - return &AuthInfo{Username: username, Revision: revision}, true + return &AuthInfo{Username: username, Revision: uint64(revision)}, true } func (t *tokenJWT) assign(ctx context.Context, username string, revision uint64) (string, error) { diff --git a/server/auth/jwt_test.go b/server/auth/jwt_test.go index a3983cc5a56a..df431c30b095 100644 --- a/server/auth/jwt_test.go +++ b/server/auth/jwt_test.go @@ -18,7 +18,10 @@ import ( "context" "fmt" "testing" + "time" + "github.com/golang-jwt/jwt/v4" + "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -202,3 +205,75 @@ func TestJWTBad(t *testing.T) { func testJWTOpts() string { return fmt.Sprintf("%s,pub-key=%s,priv-key=%s,sign-method=RS256", tokenTypeJWT, jwtRSAPubKey, jwtRSAPrivKey) } + +func TestJWTTokenWithMissingFields(t *testing.T) { + testCases := []struct { + name string + username string // An empty string means not present + revision uint64 // 0 means not present + expectValid bool + }{ + { + name: "valid token", + username: "hello", + revision: 100, + expectValid: true, + }, + { + name: "no username", + username: "", + revision: 100, + expectValid: false, + }, + { + name: "no revision", + username: "hello", + revision: 0, + expectValid: false, + }, + } + + for _, tc := range testCases { + tc := tc + optsMap := map[string]string{ + "priv-key": jwtRSAPrivKey, + "sign-method": "RS256", + "ttl": "1h", + } + + t.Run(tc.name, func(t *testing.T) { + // prepare claims + claims := jwt.MapClaims{ + "exp": time.Now().Add(time.Hour).Unix(), + } + if tc.username != "" { + claims["username"] = tc.username + } + if tc.revision != 0 { + claims["revision"] = tc.revision + } + + // generate a JWT token with the given claims + var opts jwtOptions + err := opts.ParseWithDefaults(optsMap) + require.NoError(t, err) + key, err := opts.Key() + require.NoError(t, err) + + tk := jwt.NewWithClaims(opts.SignMethod, claims) + token, err := tk.SignedString(key) + require.NoError(t, err) + + // verify the token + jwtProvider, err := newTokenProviderJWT(zap.NewNop(), optsMap) + require.NoError(t, err) + ai, ok := jwtProvider.info(context.TODO(), token, 123) + + require.Equal(t, tc.expectValid, ok) + if ok { + require.Equal(t, tc.username, ai.Username) + require.Equal(t, tc.revision, ai.Revision) + } + }) + } +} diff --git a/server/auth/options.go b/server/auth/options.go index c0b039f759c2..0b01d2c4a31d 100644 --- a/server/auth/options.go +++ b/server/auth/options.go @@ -21,7 +21,7 @@ import ( "io/ioutil" "time" - jwt "github.com/form3tech-oss/jwt-go" + "github.com/golang-jwt/jwt/v4" ) const ( diff --git a/server/auth/range_perm_cache.go b/server/auth/range_perm_cache.go index 7d77b16ea1a4..e096710c32e9 100644 --- a/server/auth/range_perm_cache.go +++ b/server/auth/range_perm_cache.go @@ -22,7 +22,7 @@ import ( "go.uber.org/zap" ) -func getMergedPerms(lg *zap.Logger, tx backend.BatchTx, userName string) *unifiedRangePermissions { +func getMergedPerms(lg *zap.Logger, tx backend.ReadTx, userName string) *unifiedRangePermissions { user := getUser(lg, tx, userName) if user == nil { return nil @@ -76,8 +76,10 @@ func checkKeyInterval( cachedPerms *unifiedRangePermissions, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool { - if len(rangeEnd) == 1 && rangeEnd[0] == 0 { + if isOpenEnded(rangeEnd) { rangeEnd = nil + // nil rangeEnd will be converetd to []byte{}, the largest element of BytesAffineComparable, + // in NewBytesAffineInterval(). } ivl := adt.NewBytesAffineInterval(key, rangeEnd) @@ -105,37 +107,99 @@ func checkKeyPoint(lg *zap.Logger, cachedPerms *unifiedRangePermissions, key []b return false } -func (as *authStore) isRangeOpPermitted(tx backend.BatchTx, userName string, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool { - // assumption: tx is Lock()ed - _, ok := as.rangePermCache[userName] +func (as *authStore) isRangeOpPermitted(userName string, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool { + as.rangePermCacheMu.RLock() + defer as.rangePermCacheMu.RUnlock() + + rangePerm, ok := as.rangePermCache[userName] if !ok { + as.lg.Error( + "user doesn't exist", + zap.String("user-name", userName), + ) + return false + } + + if len(rangeEnd) == 0 { + return checkKeyPoint(as.lg, rangePerm, key, permtyp) + } + + return checkKeyInterval(as.lg, rangePerm, key, rangeEnd, permtyp) +} + +func (as *authStore) refreshRangePermCache(tx backend.ReadTx) { + // Note that every authentication configuration update calls this method and it invalidates the entire + // rangePermCache and reconstruct it based on information of users and roles stored in the backend. + // This can be a costly operation. + as.rangePermCacheMu.Lock() + defer as.rangePermCacheMu.Unlock() + + as.rangePermCache = make(map[string]*unifiedRangePermissions) + + users := getAllUsers(as.lg, tx) + for _, user := range users { + userName := string(user.Name) perms := getMergedPerms(as.lg, tx, userName) if perms == nil { as.lg.Error( "failed to create a merged permission", zap.String("user-name", userName), ) - return false + continue } as.rangePermCache[userName] = perms } - - if len(rangeEnd) == 0 { - return checkKeyPoint(as.lg, as.rangePermCache[userName], key, permtyp) - } - - return checkKeyInterval(as.lg, as.rangePermCache[userName], key, rangeEnd, permtyp) } -func (as *authStore) clearCachedPerm() { - as.rangePermCache = make(map[string]*unifiedRangePermissions) +type unifiedRangePermissions struct { + readPerms adt.IntervalTree + writePerms adt.IntervalTree } -func (as *authStore) invalidateCachedPerm(userName string) { - delete(as.rangePermCache, userName) +// Constraints related to key range +// Assumptions: +// a1. key must be non-nil +// a2. []byte{} (in the case of string, "") is not a valid key of etcd +// For representing an open-ended range, BytesAffineComparable uses []byte{} as the largest element. +// a3. []byte{0x00} is the minimum valid etcd key +// +// Based on the above assumptions, key and rangeEnd must follow below rules: +// b1. for representing a single key point, rangeEnd should be nil or zero length byte array (in the case of string, "") +// Rule a2 guarantees that (X, []byte{}) for any X is not a valid range. So such ranges can be used for representing +// a single key permission. +// +// b2. key range with upper limit, like (X, Y), larger or equal to X and smaller than Y +// +// b3. key range with open-ended, like (X, ), is represented like (X, []byte{0x00}) +// Because of rule a3, if we have (X, []byte{0x00}), such a range represents an empty range and makes no sense to have +// such a permission. So we use []byte{0x00} for representing an open-ended permission. +// Note that rangeEnd with []byte{0x00} will be converted into []byte{} before inserted into the interval tree +// (rule a2 ensures that this is the largest element). +// Special range like key = []byte{0x00} and rangeEnd = []byte{0x00} is treated as a range which matches with all keys. +// +// Treating a range whose rangeEnd with []byte{0x00} as an open-ended comes from the rules of Range() and Watch() API. + +func isOpenEnded(rangeEnd []byte) bool { // check rule b3 + return len(rangeEnd) == 1 && rangeEnd[0] == 0 } -type unifiedRangePermissions struct { - readPerms adt.IntervalTree - writePerms adt.IntervalTree +func isValidPermissionRange(key, rangeEnd []byte) bool { + if len(key) == 0 { + return false + } + if rangeEnd == nil || len(rangeEnd) == 0 { // ensure rule b1 + return true + } + + begin := adt.BytesAffineComparable(key) + end := adt.BytesAffineComparable(rangeEnd) + if begin.Compare(end) == -1 { // rule b2 + return true + } + + if isOpenEnded(rangeEnd) { + return true + } + + return false } diff --git a/server/auth/range_perm_cache_test.go b/server/auth/range_perm_cache_test.go index 2f325159309c..b8859ee48f4b 100644 --- a/server/auth/range_perm_cache_test.go +++ b/server/auth/range_perm_cache_test.go @@ -45,6 +45,26 @@ func TestRangePermission(t *testing.T) { []byte("a"), []byte("f"), true, }, + { + []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte("f"))}, + []byte("a"), []byte{}, + false, + }, + { + []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte{})}, + []byte("a"), []byte{}, + true, + }, + { + []adt.Interval{adt.NewBytesAffineInterval([]byte{0x00}, []byte{})}, + []byte("a"), []byte{}, + true, + }, + { + []adt.Interval{adt.NewBytesAffineInterval([]byte{0x00}, []byte{})}, + []byte{0x00}, []byte{}, + true, + }, } for i, tt := range tests { @@ -86,6 +106,16 @@ func TestKeyPermission(t *testing.T) { []byte("f"), false, }, + { + []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte{})}, + []byte("f"), + true, + }, + { + []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte{0x00}, []byte{})}, + []byte("f"), + true, + }, } for i, tt := range tests { @@ -100,3 +130,88 @@ func TestKeyPermission(t *testing.T) { } } } + +func TestRangeCheck(t *testing.T) { + tests := []struct { + name string + key []byte + rangeEnd []byte + want bool + }{ + { + name: "valid single key", + key: []byte("a"), + rangeEnd: []byte(""), + want: true, + }, + { + name: "valid single key", + key: []byte("a"), + rangeEnd: nil, + want: true, + }, + { + name: "valid key range, key < rangeEnd", + key: []byte("a"), + rangeEnd: []byte("b"), + want: true, + }, + { + name: "invalid empty key range, key == rangeEnd", + key: []byte("a"), + rangeEnd: []byte("a"), + want: false, + }, + { + name: "invalid empty key range, key > rangeEnd", + key: []byte("b"), + rangeEnd: []byte("a"), + want: false, + }, + { + name: "invalid key, key must not be \"\"", + key: []byte(""), + rangeEnd: []byte("a"), + want: false, + }, + { + name: "invalid key range, key must not be \"\"", + key: []byte(""), + rangeEnd: []byte(""), + want: false, + }, + { + name: "invalid key range, key must not be \"\"", + key: []byte(""), + rangeEnd: []byte("\x00"), + want: false, + }, + { + name: "valid single key (not useful in practice)", + key: []byte("\x00"), + rangeEnd: []byte(""), + want: true, + }, + { + name: "valid key range, larger or equals to \"a\"", + key: []byte("a"), + rangeEnd: []byte("\x00"), + want: true, + }, + { + name: "valid key range, which includes all keys", + key: []byte("\x00"), + rangeEnd: []byte("\x00"), + want: true, + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isValidPermissionRange(tt.key, tt.rangeEnd) + if result != tt.want { + t.Errorf("#%d: result=%t, want=%t", i, result, tt.want) + } + }) + } +} diff --git a/server/auth/simple_token.go b/server/auth/simple_token.go index 7b1b094ae140..a657d9daefc8 100644 --- a/server/auth/simple_token.go +++ b/server/auth/simple_token.go @@ -156,13 +156,18 @@ func (t *tokenSimple) invalidateUser(username string) { } func (t *tokenSimple) enable() { + t.simpleTokensMu.Lock() + defer t.simpleTokensMu.Unlock() + if t.simpleTokenKeeper != nil { // already enabled + return + } if t.simpleTokenTTL <= 0 { t.simpleTokenTTL = simpleTokenTTLDefault } delf := func(tk string) { if username, ok := t.simpleTokens[tk]; ok { - t.lg.Info( + t.lg.Debug( "deleted a simple token", zap.String("user-name", username), zap.String("token", tk), diff --git a/server/auth/store.go b/server/auth/store.go index 19dd7e738ad6..67c5694cde31 100644 --- a/server/auth/store.go +++ b/server/auth/store.go @@ -30,6 +30,7 @@ import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/mvcc/buckets" "go.uber.org/zap" "golang.org/x/crypto/bcrypt" @@ -45,10 +46,6 @@ var ( revisionKey = []byte("authRevision") - authBucketName = []byte("auth") - authUsersBucketName = []byte("authUsers") - authRolesBucketName = []byte("authRoles") - ErrRootUserNotExist = errors.New("auth: root user does not exist") ErrRootRoleNotExist = errors.New("auth: root user does not have root role") ErrUserAlreadyExist = errors.New("auth: user already exists") @@ -57,6 +54,7 @@ var ( ErrRoleAlreadyExist = errors.New("auth: role already exists") ErrRoleNotFound = errors.New("auth: role not found") ErrRoleEmpty = errors.New("auth: role name is empty") + ErrPermissionNotGiven = errors.New("auth: permission not given") ErrAuthFailed = errors.New("auth: authentication failed, invalid user ID or password") ErrNoPasswordUser = errors.New("auth: authentication failed, password was given for no password user") ErrPermissionDenied = errors.New("auth: permission denied") @@ -210,7 +208,14 @@ type authStore struct { enabled bool enabledMu sync.RWMutex - rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions + // rangePermCache needs to be protected by rangePermCacheMu + // rangePermCacheMu needs to be write locked only in initialization phase or configuration changes + // Hot paths like Range(), needs to acquire read lock for improving performance + // + // Note that BatchTx and ReadTx cannot be a mutex for rangePermCache because they are independent resources + // see also: https://github.com/etcd-io/etcd/pull/13920#discussion_r849114855 + rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions + rangePermCacheMu sync.RWMutex tokenProvider TokenProvider bcryptCost int // the algorithm cost / strength for hashing auth passwords @@ -225,7 +230,7 @@ func (as *authStore) AuthEnable() error { } b := as.be tx := b.BatchTx() - tx.Lock() + tx.LockInsideApply() defer func() { tx.Unlock() b.ForceCommit() @@ -240,12 +245,12 @@ func (as *authStore) AuthEnable() error { return ErrRootRoleNotExist } - tx.UnsafePut(authBucketName, enableFlagKey, authEnabled) + tx.UnsafePut(buckets.Auth, enableFlagKey, authEnabled) as.enabled = true as.tokenProvider.enable() - as.rangePermCache = make(map[string]*unifiedRangePermissions) + as.refreshRangePermCache(tx) as.setRevision(getRevision(tx)) @@ -261,8 +266,8 @@ func (as *authStore) AuthDisable() { } b := as.be tx := b.BatchTx() - tx.Lock() - tx.UnsafePut(authBucketName, enableFlagKey, authDisabled) + tx.LockInsideApply() + tx.UnsafePut(buckets.Auth, enableFlagKey, authDisabled) as.commitRevision(tx) tx.Unlock() b.ForceCommit() @@ -289,7 +294,7 @@ func (as *authStore) Authenticate(ctx context.Context, username, password string } tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() user := getUser(as.lg, tx, username) @@ -326,7 +331,7 @@ func (as *authStore) CheckPassword(username, password string) (uint64, error) { // CompareHashAndPassword is very expensive, so we use closures // to avoid putting it in the critical section of the tx lock. revision, err := func() (uint64, error) { - tx := as.be.BatchTx() + tx := as.be.ReadTx() tx.Lock() defer tx.Unlock() @@ -355,9 +360,9 @@ func (as *authStore) CheckPassword(username, password string) (uint64, error) { func (as *authStore) Recover(be backend.Backend) { enabled := false as.be = be - tx := be.BatchTx() + tx := be.ReadTx() tx.Lock() - _, vs := tx.UnsafeRange(authBucketName, enableFlagKey, nil, 0) + _, vs := tx.UnsafeRange(buckets.Auth, enableFlagKey, nil, 0) if len(vs) == 1 { if bytes.Equal(vs[0], authEnabled) { enabled = true @@ -365,11 +370,15 @@ func (as *authStore) Recover(be backend.Backend) { } as.setRevision(getRevision(tx)) + as.refreshRangePermCache(tx) tx.Unlock() as.enabledMu.Lock() as.enabled = enabled + if enabled { + as.tokenProvider.enable() + } as.enabledMu.Unlock() } @@ -387,7 +396,7 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, } tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() user := getUser(as.lg, tx, r.Name) @@ -421,6 +430,7 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, putUser(as.lg, tx, newUser) as.commitRevision(tx) + as.refreshRangePermCache(tx) as.lg.Info("added a user", zap.String("user-name", r.Name)) return &pb.AuthUserAddResponse{}, nil @@ -433,7 +443,7 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete } tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() user := getUser(as.lg, tx, r.Name) @@ -444,8 +454,8 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete delUser(tx, r.Name) as.commitRevision(tx) + as.refreshRangePermCache(tx) - as.invalidateCachedPerm(r.Name) as.tokenProvider.invalidateUser(r.Name) as.lg.Info( @@ -458,7 +468,7 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() user := getUser(as.lg, tx, r.Name) @@ -469,7 +479,8 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p var password []byte var err error - if !user.Options.NoPassword { + // Backward compatible with old versions of etcd, user options is nil + if user.Options == nil || !user.Options.NoPassword { password, err = as.selectPassword(r.Password, r.HashedPassword) if err != nil { return nil, ErrNoPasswordUser @@ -486,8 +497,8 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p putUser(as.lg, tx, updatedUser) as.commitRevision(tx) + as.refreshRangePermCache(tx) - as.invalidateCachedPerm(r.Name) as.tokenProvider.invalidateUser(r.Name) as.lg.Info( @@ -500,7 +511,7 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() user := getUser(as.lg, tx, r.User) @@ -531,9 +542,8 @@ func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUser putUser(as.lg, tx, user) - as.invalidateCachedPerm(r.User) - as.commitRevision(tx) + as.refreshRangePermCache(tx) as.lg.Info( "granted a role to a user", @@ -546,7 +556,7 @@ func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUser func (as *authStore) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() user := getUser(as.lg, tx, r.Name) tx.Unlock() @@ -561,7 +571,7 @@ func (as *authStore) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, func (as *authStore) UserList(r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() users := getAllUsers(as.lg, tx) tx.Unlock() @@ -583,7 +593,7 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs } tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() user := getUser(as.lg, tx, r.Name) @@ -609,9 +619,8 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs putUser(as.lg, tx, updatedUser) - as.invalidateCachedPerm(r.Name) - as.commitRevision(tx) + as.refreshRangePermCache(tx) as.lg.Info( "revoked a role from a user", @@ -625,7 +634,7 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs func (as *authStore) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() var resp pb.AuthRoleGetResponse @@ -640,7 +649,7 @@ func (as *authStore) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, func (as *authStore) RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() roles := getAllRoles(as.lg, tx) tx.Unlock() @@ -653,7 +662,7 @@ func (as *authStore) RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListRespon func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() role := getRole(as.lg, tx, r.Role) @@ -677,11 +686,8 @@ func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest) putRole(as.lg, tx, updatedRole) - // TODO(mitake): currently single role update invalidates every cache - // It should be optimized. - as.clearCachedPerm() - as.commitRevision(tx) + as.refreshRangePermCache(tx) as.lg.Info( "revoked a permission on range", @@ -699,7 +705,7 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete } tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() role := getRole(as.lg, tx, r.Role) @@ -729,10 +735,10 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete putUser(as.lg, tx, updatedUser) - as.invalidateCachedPerm(string(user.Name)) } as.commitRevision(tx) + as.refreshRangePermCache(tx) as.lg.Info("deleted a role", zap.String("role-name", r.Role)) return &pb.AuthRoleDeleteResponse{}, nil @@ -744,7 +750,7 @@ func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, } tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() role := getRole(as.lg, tx, r.Name) @@ -783,8 +789,15 @@ func (perms permSlice) Swap(i, j int) { } func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { + if r.Perm == nil { + return nil, ErrPermissionNotGiven + } + if !isValidPermissionRange(r.Perm.Key, r.Perm.RangeEnd) { + return nil, ErrInvalidAuthMgmt + } + tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() defer tx.Unlock() role := getRole(as.lg, tx, r.Name) @@ -813,11 +826,8 @@ func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) ( putRole(as.lg, tx, role) - // TODO(mitake): currently single role update invalidates every cache - // It should be optimized. - as.clearCachedPerm() - as.commitRevision(tx) + as.refreshRangePermCache(tx) as.lg.Info( "granted/updated a permission to a user", @@ -847,7 +857,7 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE return ErrAuthOldRevision } - tx := as.be.BatchTx() + tx := as.be.ReadTx() tx.Lock() defer tx.Unlock() @@ -862,7 +872,7 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE return nil } - if as.isRangeOpPermitted(tx, userName, key, rangeEnd, permTyp) { + if as.isRangeOpPermitted(userName, key, rangeEnd, permTyp) { return nil } @@ -889,7 +899,7 @@ func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error { return ErrUserEmpty } - tx := as.be.BatchTx() + tx := as.be.ReadTx() tx.Lock() u := getUser(as.lg, tx, authInfo.Username) tx.Unlock() @@ -905,8 +915,8 @@ func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error { return nil } -func getUser(lg *zap.Logger, tx backend.BatchTx, username string) *authpb.User { - _, vs := tx.UnsafeRange(authUsersBucketName, []byte(username), nil, 0) +func getUser(lg *zap.Logger, tx backend.ReadTx, username string) *authpb.User { + _, vs := tx.UnsafeRange(buckets.AuthUsers, []byte(username), nil, 0) if len(vs) == 0 { return nil } @@ -923,8 +933,16 @@ func getUser(lg *zap.Logger, tx backend.BatchTx, username string) *authpb.User { return user } -func getAllUsers(lg *zap.Logger, tx backend.BatchTx) []*authpb.User { - _, vs := tx.UnsafeRange(authUsersBucketName, []byte{0}, []byte{0xff}, -1) +func getAllUsers(lg *zap.Logger, tx backend.ReadTx) []*authpb.User { + var vs [][]byte + err := tx.UnsafeForEach(buckets.AuthUsers, func(k []byte, v []byte) error { + vs = append(vs, v) + return nil + }) + if err != nil { + lg.Panic("failed to get users", + zap.Error(err)) + } if len(vs) == 0 { return nil } @@ -946,15 +964,15 @@ func putUser(lg *zap.Logger, tx backend.BatchTx, user *authpb.User) { if err != nil { lg.Panic("failed to unmarshal 'authpb.User'", zap.Error(err)) } - tx.UnsafePut(authUsersBucketName, user.Name, b) + tx.UnsafePut(buckets.AuthUsers, user.Name, b) } func delUser(tx backend.BatchTx, username string) { - tx.UnsafeDelete(authUsersBucketName, []byte(username)) + tx.UnsafeDelete(buckets.AuthUsers, []byte(username)) } -func getRole(lg *zap.Logger, tx backend.BatchTx, rolename string) *authpb.Role { - _, vs := tx.UnsafeRange(authRolesBucketName, []byte(rolename), nil, 0) +func getRole(lg *zap.Logger, tx backend.ReadTx, rolename string) *authpb.Role { + _, vs := tx.UnsafeRange(buckets.AuthRoles, []byte(rolename), nil, 0) if len(vs) == 0 { return nil } @@ -967,8 +985,8 @@ func getRole(lg *zap.Logger, tx backend.BatchTx, rolename string) *authpb.Role { return role } -func getAllRoles(lg *zap.Logger, tx backend.BatchTx) []*authpb.Role { - _, vs := tx.UnsafeRange(authRolesBucketName, []byte{0}, []byte{0xff}, -1) +func getAllRoles(lg *zap.Logger, tx backend.ReadTx) []*authpb.Role { + _, vs := tx.UnsafeRange(buckets.AuthRoles, []byte{0}, []byte{0xff}, -1) if len(vs) == 0 { return nil } @@ -995,11 +1013,11 @@ func putRole(lg *zap.Logger, tx backend.BatchTx, role *authpb.Role) { ) } - tx.UnsafePut(authRolesBucketName, role.Name, b) + tx.UnsafePut(buckets.AuthRoles, role.Name, b) } func delRole(tx backend.BatchTx, rolename string) { - tx.UnsafeDelete(authRolesBucketName, []byte(rolename)) + tx.UnsafeDelete(buckets.AuthRoles, []byte(rolename)) } func (as *authStore) IsAuthEnabled() bool { @@ -1026,14 +1044,14 @@ func NewAuthStore(lg *zap.Logger, be backend.Backend, tp TokenProvider, bcryptCo } tx := be.BatchTx() - tx.Lock() + tx.LockOutsideApply() - tx.UnsafeCreateBucket(authBucketName) - tx.UnsafeCreateBucket(authUsersBucketName) - tx.UnsafeCreateBucket(authRolesBucketName) + tx.UnsafeCreateBucket(buckets.Auth) + tx.UnsafeCreateBucket(buckets.AuthUsers) + tx.UnsafeCreateBucket(buckets.AuthRoles) enabled := false - _, vs := tx.UnsafeRange(authBucketName, enableFlagKey, nil, 0) + _, vs := tx.UnsafeRange(buckets.Auth, enableFlagKey, nil, 0) if len(vs) == 1 { if bytes.Equal(vs[0], authEnabled) { enabled = true @@ -1060,6 +1078,8 @@ func NewAuthStore(lg *zap.Logger, be backend.Backend, tp TokenProvider, bcryptCo as.setupMetricsReporter() + as.refreshRangePermCache(tx) + tx.Unlock() be.ForceCommit() @@ -1076,11 +1096,11 @@ func (as *authStore) commitRevision(tx backend.BatchTx) { atomic.AddUint64(&as.revision, 1) revBytes := make([]byte, revBytesLen) binary.BigEndian.PutUint64(revBytes, as.Revision()) - tx.UnsafePut(authBucketName, revisionKey, revBytes) + tx.UnsafePut(buckets.Auth, revisionKey, revBytes) } -func getRevision(tx backend.BatchTx) uint64 { - _, vs := tx.UnsafeRange(authBucketName, revisionKey, nil, 0) +func getRevision(tx backend.ReadTx) uint64 { + _, vs := tx.UnsafeRange(buckets.Auth, revisionKey, nil, 0) if len(vs) != 1 { // this can happen in the initialization phase return 0 @@ -1140,6 +1160,9 @@ func (as *authStore) AuthInfoFromTLS(ctx context.Context) (ai *AuthInfo) { } func (as *authStore) AuthInfoFromCtx(ctx context.Context) (*AuthInfo, error) { + if !as.IsAuthEnabled() { + return nil, nil + } md, ok := metadata.FromIncomingContext(ctx) if !ok { return nil, nil @@ -1279,7 +1302,7 @@ func (as *authStore) WithRoot(ctx context.Context) context.Context { func (as *authStore) HasRole(user, role string) bool { tx := as.be.BatchTx() - tx.Lock() + tx.LockInsideApply() u := getUser(as.lg, tx, user) tx.Unlock() diff --git a/server/auth/store_test.go b/server/auth/store_test.go index c530ffe8861d..46aba50d9b24 100644 --- a/server/auth/store_test.go +++ b/server/auth/store_test.go @@ -17,6 +17,7 @@ package auth import ( "context" "encoding/base64" + "errors" "fmt" "reflect" "strings" @@ -27,6 +28,7 @@ import ( "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + "go.etcd.io/etcd/pkg/v3/adt" "go.etcd.io/etcd/server/v3/mvcc/backend" betesting "go.etcd.io/etcd/server/v3/mvcc/backend/testing" @@ -73,7 +75,7 @@ func TestNewAuthStoreRevision(t *testing.T) { } } -// TestNewAuthStoreBryptCost ensures that NewAuthStore uses default when given bcrypt-cost is invalid +// TestNewAuthStoreBcryptCost ensures that NewAuthStore uses default when given bcrypt-cost is invalid func TestNewAuthStoreBcryptCost(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, b) @@ -123,6 +125,13 @@ func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testin t.Fatal(err) } + // The UserAdd function cannot generate old etcd version user data (user's option is nil) + // add special users through the underlying interface + err = addUserWithNoOption(as) + if err != nil { + t.Fatal(err) + } + tearDown := func(_ *testing.T) { b.Close() as.Close() @@ -130,6 +139,14 @@ func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testin return as, tearDown } +func addUserWithNoOption(as *authStore) error { + _, err := as.UserAdd(&pb.AuthUserAddRequest{Name: "foo-no-user-options", Password: "bar"}) + if err != nil { + return err + } + return nil +} + func enableAuthAndCreateRoot(as *authStore) error { _, err := as.UserAdd(&pb.AuthUserAddRequest{Name: "root", HashedPassword: encodePassword("root"), Options: &authpb.UserAddOptions{NoPassword: false}}) if err != nil { @@ -153,7 +170,8 @@ func TestUserAdd(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) - ua := &pb.AuthUserAddRequest{Name: "foo", Options: &authpb.UserAddOptions{NoPassword: false}} + const userName = "foo" + ua := &pb.AuthUserAddRequest{Name: userName, Options: &authpb.UserAddOptions{NoPassword: false}} _, err := as.UserAdd(ua) // add an existing user if err == nil { t.Fatalf("expected %v, got %v", ErrUserAlreadyExist, err) @@ -167,6 +185,11 @@ func TestUserAdd(t *testing.T) { if err != ErrUserEmpty { t.Fatal(err) } + + if _, ok := as.rangePermCache[userName]; !ok { + t.Fatalf("user %s should be added but it doesn't exist in rangePermCache", userName) + + } } func TestRecover(t *testing.T) { @@ -182,6 +205,30 @@ func TestRecover(t *testing.T) { } } +func TestRecoverWithEmptyRangePermCache(t *testing.T) { + as, tearDown := setupAuthStore(t) + defer as.Close() + defer tearDown(t) + + as.enabled = false + as.rangePermCache = map[string]*unifiedRangePermissions{} + as.Recover(as.be) + + if !as.IsAuthEnabled() { + t.Fatalf("expected auth enabled got disabled") + } + + if len(as.rangePermCache) != 3 { + t.Fatalf("rangePermCache should have permission information for 3 users (\"root\" and \"foo\",\"foo-no-user-options\"), but has %d information", len(as.rangePermCache)) + } + if _, ok := as.rangePermCache["root"]; !ok { + t.Fatal("user \"root\" should be created by setupAuthStore() but doesn't exist in rangePermCache") + } + if _, ok := as.rangePermCache["foo"]; !ok { + t.Fatal("user \"foo\" should be created by setupAuthStore() but doesn't exist in rangePermCache") + } +} + func TestCheckPassword(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) @@ -216,7 +263,8 @@ func TestUserDelete(t *testing.T) { defer tearDown(t) // delete an existing user - ud := &pb.AuthUserDeleteRequest{Name: "foo"} + const userName = "foo" + ud := &pb.AuthUserDeleteRequest{Name: userName} _, err := as.UserDelete(ud) if err != nil { t.Fatal(err) @@ -230,6 +278,47 @@ func TestUserDelete(t *testing.T) { if err != ErrUserNotFound { t.Fatalf("expected %v, got %v", ErrUserNotFound, err) } + + if _, ok := as.rangePermCache[userName]; ok { + t.Fatalf("user %s should be deleted but it exists in rangePermCache", userName) + + } +} + +func TestUserDeleteAndPermCache(t *testing.T) { + as, tearDown := setupAuthStore(t) + defer tearDown(t) + + // delete an existing user + const deletedUserName = "foo" + ud := &pb.AuthUserDeleteRequest{Name: deletedUserName} + _, err := as.UserDelete(ud) + if err != nil { + t.Fatal(err) + } + + // delete a non-existing user + _, err = as.UserDelete(ud) + if err != ErrUserNotFound { + t.Fatalf("expected %v, got %v", ErrUserNotFound, err) + } + + if _, ok := as.rangePermCache[deletedUserName]; ok { + t.Fatalf("user %s should be deleted but it exists in rangePermCache", deletedUserName) + } + + // add a new user + const newUser = "bar" + ua := &pb.AuthUserAddRequest{Name: newUser, HashedPassword: encodePassword("pwd1"), Options: &authpb.UserAddOptions{NoPassword: false}} + _, err = as.UserAdd(ua) + if err != nil { + t.Fatal(err) + } + + if _, ok := as.rangePermCache[newUser]; !ok { + t.Fatalf("user %s should exist but it doesn't exist in rangePermCache", deletedUserName) + + } } func TestUserChangePassword(t *testing.T) { @@ -261,6 +350,12 @@ func TestUserChangePassword(t *testing.T) { if err != ErrUserNotFound { t.Fatalf("expected %v, got %v", ErrUserNotFound, err) } + + // change a user(user option is nil) password + _, err = as.UserChangePassword(&pb.AuthUserChangePasswordRequest{Name: "foo-no-user-options", HashedPassword: encodePassword("bar")}) + if err != nil { + t.Fatal(err) + } } func TestRoleAdd(t *testing.T) { @@ -448,6 +543,162 @@ func TestRoleGrantPermission(t *testing.T) { if !reflect.DeepEqual(perm, r.Perm[0]) { t.Errorf("expected %v, got %v", perm, r.Perm[0]) } + + // trying to grant nil permissions returns an error (and doesn't change the actual permissions!) + _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ + Name: "role-test-1", + }) + + if err != ErrPermissionNotGiven { + t.Error(err) + } + + r, err = as.RoleGet(&pb.AuthRoleGetRequest{Role: "role-test-1"}) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(perm, r.Perm[0]) { + t.Errorf("expected %v, got %v", perm, r.Perm[0]) + } +} + +func TestRoleGrantInvalidPermission(t *testing.T) { + as, tearDown := setupAuthStore(t) + defer tearDown(t) + + _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + perm *authpb.Permission + want error + }{ + { + name: "valid range", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("Keys"), + RangeEnd: []byte("RangeEnd"), + }, + want: nil, + }, + { + name: "invalid range: nil key", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: nil, + RangeEnd: []byte("RangeEnd"), + }, + want: ErrInvalidAuthMgmt, + }, + { + name: "valid range: single key", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("Keys"), + RangeEnd: nil, + }, + want: nil, + }, + { + name: "valid range: single key", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("Keys"), + RangeEnd: []byte{}, + }, + want: nil, + }, + { + name: "invalid range: empty (Key == RangeEnd)", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("a"), + RangeEnd: []byte("a"), + }, + want: ErrInvalidAuthMgmt, + }, + { + name: "invalid range: empty (Key > RangeEnd)", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("b"), + RangeEnd: []byte("a"), + }, + want: ErrInvalidAuthMgmt, + }, + { + name: "invalid range: length of key is 0", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte(""), + RangeEnd: []byte("a"), + }, + want: ErrInvalidAuthMgmt, + }, + { + name: "invalid range: length of key is 0", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte(""), + RangeEnd: []byte(""), + }, + want: ErrInvalidAuthMgmt, + }, + { + name: "invalid range: length of key is 0", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte(""), + RangeEnd: []byte{0x00}, + }, + want: ErrInvalidAuthMgmt, + }, + { + name: "valid range: single key permission for []byte{0x00}", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte{0x00}, + RangeEnd: []byte(""), + }, + want: nil, + }, + { + name: "valid range: \"a\" or larger keys", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("a"), + RangeEnd: []byte{0x00}, + }, + want: nil, + }, + { + name: "valid range: the entire keys", + perm: &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte{0x00}, + RangeEnd: []byte{0x00}, + }, + want: nil, + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ + Name: "role-test-1", + Perm: tt.perm, + }) + + if !errors.Is(err, tt.want) { + t.Errorf("#%d: result=%t, want=%t", i, err, tt.want) + } + }) + } } func TestRoleRevokePermission(t *testing.T) { @@ -506,17 +757,44 @@ func TestUserRevokePermission(t *testing.T) { t.Fatal(err) } - _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test"}) + const userName = "foo" + _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userName, Role: "role-test"}) if err != nil { t.Fatal(err) } - _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test-1"}) + _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userName, Role: "role-test-1"}) if err != nil { t.Fatal(err) } - u, err := as.UserGet(&pb.AuthUserGetRequest{Name: "foo"}) + perm := &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("WriteKeyBegin"), + RangeEnd: []byte("WriteKeyEnd"), + } + _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ + Name: "role-test-1", + Perm: perm, + }) + if err != nil { + t.Fatal(err) + } + + if _, ok := as.rangePermCache[userName]; !ok { + t.Fatalf("User %s should have its entry in rangePermCache", userName) + } + unifiedPerm := as.rangePermCache[userName] + pt1 := adt.NewBytesAffinePoint([]byte("WriteKeyBegin")) + if !unifiedPerm.writePerms.Contains(pt1) { + t.Fatal("rangePermCache should contain WriteKeyBegin") + } + pt2 := adt.NewBytesAffinePoint([]byte("OutOfRange")) + if unifiedPerm.writePerms.Contains(pt2) { + t.Fatal("rangePermCache should not contain OutOfRange") + } + + u, err := as.UserGet(&pb.AuthUserGetRequest{Name: userName}) if err != nil { t.Fatal(err) } @@ -526,12 +804,12 @@ func TestUserRevokePermission(t *testing.T) { t.Fatalf("expected %v, got %v", expected, u.Roles) } - _, err = as.UserRevokeRole(&pb.AuthUserRevokeRoleRequest{Name: "foo", Role: "role-test-1"}) + _, err = as.UserRevokeRole(&pb.AuthUserRevokeRoleRequest{Name: userName, Role: "role-test-1"}) if err != nil { t.Fatal(err) } - u, err = as.UserGet(&pb.AuthUserGetRequest{Name: "foo"}) + u, err = as.UserGet(&pb.AuthUserGetRequest{Name: userName}) if err != nil { t.Fatal(err) } @@ -648,7 +926,7 @@ func TestIsAuthEnabled(t *testing.T) { } } -// TestAuthRevisionRace ensures that access to authStore.revision is thread-safe. +// TestAuthInfoFromCtxRace ensures that access to authStore.revision is thread-safe. func TestAuthInfoFromCtxRace(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, b) diff --git a/server/config/config.go b/server/config/config.go index b6e2109c2281..1203f7a0dfe8 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -17,8 +17,10 @@ package config import ( "context" "fmt" + "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -114,12 +116,17 @@ type ServerConfig struct { AutoCompactionRetention time.Duration AutoCompactionMode string CompactionBatchLimit int + CompactionSleepInterval time.Duration QuotaBackendBytes int64 MaxTxnOps uint // MaxRequestBytes is the maximum request size to send over raft. MaxRequestBytes uint + // MaxConcurrentStreams specifies the maximum number of concurrent + // streams that each client can open at a time. + MaxConcurrentStreams uint32 + WarningApplyDuration time.Duration StrictReconfigCheck bool @@ -133,8 +140,10 @@ type ServerConfig struct { // InitialCorruptCheck is true to check data corruption on boot // before serving any peer/client traffic. - InitialCorruptCheck bool - CorruptCheckTime time.Duration + InitialCorruptCheck bool + CorruptCheckTime time.Duration + CompactHashCheckEnabled bool + CompactHashCheckTime time.Duration // PreVote is true to enable Raft Pre-Vote. PreVote bool @@ -145,12 +154,15 @@ type ServerConfig struct { // Logger logs server-side operations. Logger *zap.Logger - ForceNewCluster bool + ForceNewCluster bool + ForceNewClusterBumpAmount uint64 - // EnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. + // EnableLeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change. EnableLeaseCheckpoint bool // LeaseCheckpointInterval time.Duration is the wait duration between lease checkpoints. LeaseCheckpointInterval time.Duration + // LeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. + LeaseCheckpointPersist bool EnableGRPCGateway bool @@ -179,12 +191,23 @@ type ServerConfig struct { // a shared buffer in its readonly check operations. ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"` + // ExperimentalStopGRPCServiceOnDefrag enables etcd gRPC service to stop serving client requests on defragmentation. + ExperimentalStopGRPCServiceOnDefrag bool `json:"experimental-stop-grpc-service-on-defrag"` + // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` + // ExperimentalMaxLearners sets a limit to the number of learner members that can exist in the cluster membership. + ExperimentalMaxLearners int `json:"experimental-max-learners"` + // V2Deprecation defines a phase of v2store deprecation process. V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` + + // openshiftHardwareDelaySeconds sets the duration, in seconds, to extend the standard etcd + // request timeout by based on hardware speed. OpenShift currently has a notion of differentiating + // between standard and slower hardware: https://github.com/openshift/cluster-etcd-operator/blob/5bbe49442101475febb89bba3be808aa121f5c0c/pkg/hwspeedhelpers/hwhelper.go#L21-L33 + openshiftHardwareDelaySeconds int } // VerifyBootstrap sanity-checks the initial config for bootstrap case @@ -299,7 +322,31 @@ func (c *ServerConfig) ShouldDiscover() bool { return c.DiscoveryURL != "" } func (c *ServerConfig) ReqTimeout() time.Duration { // 5s for queue waiting, computation and disk IO delay // + 2 * election timeout for possible leader election - return 5*time.Second + 2*time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond + // return 5*time.Second + 2*time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond + + // OCP PATCH: use an environment variable to configure the non election-related timeout + // value used. If it isn't configured, default to 25 seconds. + // Default of 25 seconds gets us to a total of 27s on "standard hardware" and 30s on "slow hardware" + // due to the configurations of election timeouts in https://github.com/openshift/cluster-etcd-operator/blob/5bbe49442101475febb89bba3be808aa121f5c0c/pkg/hwspeedhelpers/hwhelper.go#L21-L33 + // This gets us closer to the kube-apiserver's upper bound request timeout of 34 seconds while leaving some time + // for other things to have on the kube-apiserver side of things. + // See: https://issues.redhat.com/browse/OCPBUGS-50510 + // TODO(everettraven): pursue an upstream fix for this so we don't have to carry this patch for forever + if c.openshiftHardwareDelaySeconds == 0 { + hardwareDelayTimeout := 25 + if hardwareDelayOverrideEnv := os.Getenv("OPENSHIFT_ETCD_HARDWARE_DELAY_TIMEOUT"); hardwareDelayOverrideEnv != "" { + hardwareDelayOverride, err := strconv.Atoi(hardwareDelayOverrideEnv) + if err != nil { + c.Logger.Sugar().Infof("OPENSHIFT_ETCD_HARDWARE_DELAY_TIMEOUT specified but could not be parsed. falling back to default of %s seconds. parse error: %v", hardwareDelayTimeout, err) + } else { + hardwareDelayTimeout = hardwareDelayOverride + } + } + + c.openshiftHardwareDelaySeconds = hardwareDelayTimeout + } + + return time.Duration(c.openshiftHardwareDelaySeconds)*time.Second + 2*time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond } func (c *ServerConfig) ElectionTimeout() time.Duration { diff --git a/server/embed/config.go b/server/embed/config.go index 380c0c3aaa8a..5e0fe2529017 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -15,8 +15,10 @@ package embed import ( + "crypto/tls" "fmt" "io/ioutil" + "math" "net" "net/http" "net/url" @@ -35,6 +37,7 @@ import ( "go.etcd.io/etcd/pkg/v3/netutil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor" bolt "go.etcd.io/bbolt" @@ -55,6 +58,7 @@ const ( DefaultMaxTxnOps = uint(128) DefaultWarningApplyDuration = 100 * time.Millisecond DefaultMaxRequestBytes = 1.5 * 1024 * 1024 + DefaultMaxConcurrentStreams = math.MaxUint32 DefaultGRPCKeepAliveMinTime = 5 * time.Second DefaultGRPCKeepAliveInterval = 2 * time.Hour DefaultGRPCKeepAliveTimeout = 20 * time.Second @@ -148,7 +152,7 @@ type Config struct { // follower to catch up. // WARNING: only change this for tests. // Always use "DefaultSnapshotCatchUpEntries" - SnapshotCatchUpEntries uint64 + SnapshotCatchUpEntries uint64 `json:"experimental-snapshot-catch-up-entries"` MaxSnapFiles uint `json:"max-snapshots"` MaxWalFiles uint `json:"max-wals"` @@ -198,22 +202,31 @@ type Config struct { MaxTxnOps uint `json:"max-txn-ops"` MaxRequestBytes uint `json:"max-request-bytes"` - LPUrls, LCUrls []url.URL - APUrls, ACUrls []url.URL - ClientTLSInfo transport.TLSInfo - ClientAutoTLS bool - PeerTLSInfo transport.TLSInfo - PeerAutoTLS bool + // MaxConcurrentStreams specifies the maximum number of concurrent + // streams that each client can open at a time. + MaxConcurrentStreams uint32 `json:"max-concurrent-streams"` + + ListenPeerUrls, ListenClientUrls, ListenClientHttpUrls []url.URL + AdvertisePeerUrls, AdvertiseClientUrls []url.URL + ClientTLSInfo transport.TLSInfo + ClientAutoTLS bool + PeerTLSInfo transport.TLSInfo + PeerAutoTLS bool // SelfSignedCertValidity specifies the validity period of the client and peer certificates // that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS, // the unit is year, and the default is 1 - SelfSignedCertValidity uint + SelfSignedCertValidity uint `json:"self-signed-cert-validity"` // CipherSuites is a list of supported TLS cipher suites between // client/server and peers. If empty, Go auto-populates the list. // Note that cipher suites are prioritized in the given order. CipherSuites []string `json:"cipher-suites"` + // TlsMinVersion is the minimum accepted TLS version between client/server and peers. + TlsMinVersion string `json:"tls-min-version"` + // TlsMaxVersion is the maximum accepted TLS version between client/server and peers. + TlsMaxVersion string `json:"tls-max-version"` + ClusterState string `json:"initial-cluster-state"` DNSCluster string `json:"discovery-srv"` DNSClusterServiceName string `json:"discovery-srv-name"` @@ -252,7 +265,7 @@ type Config struct { GRPCKeepAliveTimeout time.Duration `json:"grpc-keepalive-timeout"` // SocketOpts are socket options passed to listener config. - SocketOpts transport.SocketOpts + SocketOpts transport.SocketOpts `json:"socket-options"` // PreVote is true to enable Raft Pre-Vote. // If enabled, Raft runs an additional election phase @@ -305,18 +318,27 @@ type Config struct { AuthToken string `json:"auth-token"` BcryptCost uint `json:"bcrypt-cost"` - //The AuthTokenTTL in seconds of the simple token + // AuthTokenTTL specifies the TTL in seconds of the simple token AuthTokenTTL uint `json:"auth-token-ttl"` - ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` - ExperimentalCorruptCheckTime time.Duration `json:"experimental-corrupt-check-time"` + ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` + ExperimentalCorruptCheckTime time.Duration `json:"experimental-corrupt-check-time"` + ExperimentalCompactHashCheckEnabled bool `json:"experimental-compact-hash-check-enabled"` + ExperimentalCompactHashCheckTime time.Duration `json:"experimental-compact-hash-check-time"` // ExperimentalEnableV2V3 configures URLs that expose deprecated V2 API working on V3 store. // Deprecated in v3.5. // TODO: Delete in v3.6 (https://github.com/etcd-io/etcd/issues/12913) ExperimentalEnableV2V3 string `json:"experimental-enable-v2v3"` - // ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. - ExperimentalEnableLeaseCheckpoint bool `json:"experimental-enable-lease-checkpoint"` - ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"` + // ExperimentalEnableLeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change. + ExperimentalEnableLeaseCheckpoint bool `json:"experimental-enable-lease-checkpoint"` + // ExperimentalEnableLeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. + // Requires experimental-enable-lease-checkpoint to be enabled. + // Deprecated in v3.6. + // TODO: Delete in v3.7 + ExperimentalEnableLeaseCheckpointPersist bool `json:"experimental-enable-lease-checkpoint-persist"` + ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"` + // ExperimentalCompactionSleepInterval is the sleep interval between every etcd compaction loop. + ExperimentalCompactionSleepInterval time.Duration `json:"experimental-compaction-sleep-interval"` ExperimentalWatchProgressNotifyInterval time.Duration `json:"experimental-watch-progress-notify-interval"` // ExperimentalWarningApplyDuration is the time duration after which a warning is generated if applying request // takes more time than this value. @@ -324,9 +346,12 @@ type Config struct { // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` + // ExperimentalMaxLearners sets a limit to the number of learner members that can exist in the cluster membership. + ExperimentalMaxLearners int `json:"experimental-max-learners"` // ForceNewCluster starts a new cluster even if previously started; unsafe. - ForceNewCluster bool `json:"force-new-cluster"` + ForceNewCluster bool `json:"force-new-cluster"` + ForceNewClusterBumpAmount uint64 `json:"force-new-cluster-bump-amount"` EnablePprof bool `json:"enable-pprof"` Metrics string `json:"metrics"` @@ -346,6 +371,9 @@ type Config struct { // that exist at the same time. // Can only be used if ExperimentalEnableDistributedTracing is true. ExperimentalDistributedTracingServiceInstanceID string `json:"experimental-distributed-tracing-instance-id"` + // ExperimentalDistributedTracingSamplingRatePerMillion is the number of samples to collect per million spans. + // Defaults to 0. + ExperimentalDistributedTracingSamplingRatePerMillion int `json:"experimental-distributed-tracing-sampling-rate"` // Logger is logger options: currently only supports "zap". // "capnslog" is removed in v3.5. @@ -392,6 +420,9 @@ type Config struct { // ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations. ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"` + // ExperimentalStopGRPCServiceOnDefrag enables etcd gRPC service to stop serving client requests on defragmentation. + ExperimentalStopGRPCServiceOnDefrag bool `json:"experimental-stop-grpc-service-on-defrag"` + // V2Deprecation describes phase of API & Storage V2 support V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"` } @@ -404,10 +435,11 @@ type configYAML struct { // configJSON has file options that are translated into Config options type configJSON struct { - LPUrlsJSON string `json:"listen-peer-urls"` - LCUrlsJSON string `json:"listen-client-urls"` - APUrlsJSON string `json:"initial-advertise-peer-urls"` - ACUrlsJSON string `json:"advertise-client-urls"` + ListenPeerUrls string `json:"listen-peer-urls"` + ListenClientUrls string `json:"listen-client-urls"` + ListenClientHttpUrls string `json:"listen-client-http-urls"` + AdvertisePeerUrls string `json:"initial-advertise-peer-urls"` + AdvertiseClientUrls string `json:"advertise-client-urls"` CORSJSON string `json:"cors"` HostWhitelistJSON string `json:"host-whitelist"` @@ -417,13 +449,15 @@ type configJSON struct { } type securityConfig struct { - CertFile string `json:"cert-file"` - KeyFile string `json:"key-file"` - ClientCertFile string `json:"client-cert-file"` - ClientKeyFile string `json:"client-key-file"` - CertAuth bool `json:"client-cert-auth"` - TrustedCAFile string `json:"trusted-ca-file"` - AutoTLS bool `json:"auto-tls"` + CertFile string `json:"cert-file"` + KeyFile string `json:"key-file"` + ClientCertFile string `json:"client-cert-file"` + ClientKeyFile string `json:"client-key-file"` + CertAuth bool `json:"client-cert-auth"` + TrustedCAFile string `json:"trusted-ca-file"` + AutoTLS bool `json:"auto-tls"` + AllowedCNs []string `json:"allowed-cn"` + AllowedHostnames []string `json:"allowed-hostname"` } // NewConfig creates a new Config populated with default values. @@ -443,22 +477,26 @@ func NewConfig() *Config { MaxTxnOps: DefaultMaxTxnOps, MaxRequestBytes: DefaultMaxRequestBytes, + MaxConcurrentStreams: DefaultMaxConcurrentStreams, ExperimentalWarningApplyDuration: DefaultWarningApplyDuration, GRPCKeepAliveMinTime: DefaultGRPCKeepAliveMinTime, GRPCKeepAliveInterval: DefaultGRPCKeepAliveInterval, GRPCKeepAliveTimeout: DefaultGRPCKeepAliveTimeout, - SocketOpts: transport.SocketOpts{}, + SocketOpts: transport.SocketOpts{ + ReusePort: false, + ReuseAddress: false, + }, TickMs: 100, ElectionMs: 1000, InitialElectionTickAdvance: true, - LPUrls: []url.URL{*lpurl}, - LCUrls: []url.URL{*lcurl}, - APUrls: []url.URL{*apurl}, - ACUrls: []url.URL{*acurl}, + ListenPeerUrls: []url.URL{*lpurl}, + ListenClientUrls: []url.URL{*lcurl}, + AdvertisePeerUrls: []url.URL{*apurl}, + AdvertiseClientUrls: []url.URL{*acurl}, ClusterState: ClusterStateFlagNew, InitialClusterToken: "etcd-cluster", @@ -488,6 +526,11 @@ func NewConfig() *Config { ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, ExperimentalMemoryMlock: false, ExperimentalTxnModeWriteWithSharedBuffer: true, + ExperimentalStopGRPCServiceOnDefrag: false, + ExperimentalMaxLearners: membership.DefaultMaxLearners, + + ExperimentalCompactHashCheckEnabled: false, + ExperimentalCompactHashCheckTime: time.Minute, V2Deprecation: config.V2_DEPR_DEFAULT, } @@ -516,40 +559,49 @@ func (cfg *configYAML) configFromFile(path string) error { return err } - if cfg.LPUrlsJSON != "" { - u, err := types.NewURLs(strings.Split(cfg.LPUrlsJSON, ",")) + if cfg.configJSON.ListenPeerUrls != "" { + u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenPeerUrls, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up listen-peer-urls: %v\n", err) os.Exit(1) } - cfg.LPUrls = []url.URL(u) + cfg.Config.ListenPeerUrls = u } - if cfg.LCUrlsJSON != "" { - u, err := types.NewURLs(strings.Split(cfg.LCUrlsJSON, ",")) + if cfg.configJSON.ListenClientUrls != "" { + u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenClientUrls, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-urls: %v\n", err) os.Exit(1) } - cfg.LCUrls = []url.URL(u) + cfg.Config.ListenClientUrls = u + } + + if cfg.configJSON.ListenClientHttpUrls != "" { + u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenClientHttpUrls, ",")) + if err != nil { + fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-http-urls: %v\n", err) + os.Exit(1) + } + cfg.Config.ListenClientHttpUrls = u } - if cfg.APUrlsJSON != "" { - u, err := types.NewURLs(strings.Split(cfg.APUrlsJSON, ",")) + if cfg.configJSON.AdvertisePeerUrls != "" { + u, err := types.NewURLs(strings.Split(cfg.configJSON.AdvertisePeerUrls, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up initial-advertise-peer-urls: %v\n", err) os.Exit(1) } - cfg.APUrls = []url.URL(u) + cfg.Config.AdvertisePeerUrls = u } - if cfg.ACUrlsJSON != "" { - u, err := types.NewURLs(strings.Split(cfg.ACUrlsJSON, ",")) + if cfg.configJSON.AdvertiseClientUrls != "" { + u, err := types.NewURLs(strings.Split(cfg.configJSON.AdvertiseClientUrls, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up advertise-peer-urls: %v\n", err) os.Exit(1) } - cfg.ACUrls = []url.URL(u) + cfg.Config.AdvertiseClientUrls = u } if cfg.ListenMetricsUrlsJSON != "" { @@ -586,12 +638,16 @@ func (cfg *configYAML) configFromFile(path string) error { tls.ClientKeyFile = ysc.ClientKeyFile tls.ClientCertAuth = ysc.CertAuth tls.TrustedCAFile = ysc.TrustedCAFile + tls.AllowedCNs = ysc.AllowedCNs + tls.AllowedHostnames = ysc.AllowedHostnames } copySecurityDetails(&cfg.ClientTLSInfo, &cfg.ClientSecurityJSON) copySecurityDetails(&cfg.PeerTLSInfo, &cfg.PeerSecurityJSON) cfg.ClientAutoTLS = cfg.ClientSecurityJSON.AutoTLS cfg.PeerAutoTLS = cfg.PeerSecurityJSON.AutoTLS - + if cfg.SelfSignedCertValidity == 0 { + cfg.SelfSignedCertValidity = 1 + } return cfg.Validate() } @@ -600,39 +656,52 @@ func updateCipherSuites(tls *transport.TLSInfo, ss []string) error { return fmt.Errorf("TLSInfo.CipherSuites is already specified (given %v)", ss) } if len(ss) > 0 { - cs := make([]uint16, len(ss)) - for i, s := range ss { - var ok bool - cs[i], ok = tlsutil.GetCipherSuite(s) - if !ok { - return fmt.Errorf("unexpected TLS cipher suite %q", s) - } + cs, err := tlsutil.GetCipherSuites(ss) + if err != nil { + return err } tls.CipherSuites = cs } return nil } +func updateMinMaxVersions(info *transport.TLSInfo, min, max string) { + // Validate() has been called to check the user input, so it should never fail. + var err error + if info.MinVersion, err = tlsutil.GetTLSVersion(min); err != nil { + panic(err) + } + if info.MaxVersion, err = tlsutil.GetTLSVersion(max); err != nil { + panic(err) + } +} + // Validate ensures that '*embed.Config' fields are properly configured. func (cfg *Config) Validate() error { if err := cfg.setupLogging(); err != nil { return err } - if err := checkBindURLs(cfg.LPUrls); err != nil { + if err := checkBindURLs(cfg.ListenPeerUrls); err != nil { return err } - if err := checkBindURLs(cfg.LCUrls); err != nil { + if err := checkBindURLs(cfg.ListenClientUrls); err != nil { return err } + if err := checkBindURLs(cfg.ListenClientHttpUrls); err != nil { + return err + } + if len(cfg.ListenClientHttpUrls) == 0 { + cfg.logger.Warn("Running http and grpc server on single port. This is not recommended for production.") + } if err := checkBindURLs(cfg.ListenMetricsUrls); err != nil { return err } - if err := checkHostURLs(cfg.APUrls); err != nil { - addrs := cfg.getAPURLs() + if err := checkHostURLs(cfg.AdvertisePeerUrls); err != nil { + addrs := cfg.getAdvertisePeerUrls() return fmt.Errorf(`--initial-advertise-peer-urls %q must be "host:port" (%v)`, strings.Join(addrs, ","), err) } - if err := checkHostURLs(cfg.ACUrls); err != nil { - addrs := cfg.getACURLs() + if err := checkHostURLs(cfg.AdvertiseClientUrls); err != nil { + addrs := cfg.getAdvertiseClientUrls() return fmt.Errorf(`--advertise-client-urls %q must be "host:port" (%v)`, strings.Join(addrs, ","), err) } // Check if conflicting flags are passed. @@ -665,7 +734,7 @@ func (cfg *Config) Validate() error { } // check this last since proxying in etcdmain may make this OK - if cfg.LCUrls != nil && cfg.ACUrls == nil { + if cfg.ListenClientUrls != nil && cfg.AdvertiseClientUrls == nil { return ErrUnsetAdvertiseClientURLsFlag } @@ -676,6 +745,44 @@ func (cfg *Config) Validate() error { return fmt.Errorf("unknown auto-compaction-mode %q", cfg.AutoCompactionMode) } + if !cfg.ExperimentalEnableLeaseCheckpointPersist && cfg.ExperimentalEnableLeaseCheckpoint { + cfg.logger.Warn("Detected that checkpointing is enabled without persistence. Consider enabling experimental-enable-lease-checkpoint-persist") + } + + if cfg.ExperimentalEnableLeaseCheckpointPersist && !cfg.ExperimentalEnableLeaseCheckpoint { + return fmt.Errorf("setting experimental-enable-lease-checkpoint-persist requires experimental-enable-lease-checkpoint") + } + + if cfg.ExperimentalCompactHashCheckTime <= 0 { + return fmt.Errorf("--experimental-compact-hash-check-time must be >0 (set to %v)", cfg.ExperimentalCompactHashCheckTime) + } + + minVersion, err := tlsutil.GetTLSVersion(cfg.TlsMinVersion) + if err != nil { + return err + } + maxVersion, err := tlsutil.GetTLSVersion(cfg.TlsMaxVersion) + if err != nil { + return err + } + + // maxVersion == 0 means that Go selects the highest available version. + if maxVersion != 0 && minVersion > maxVersion { + return fmt.Errorf("min version (%s) is greater than max version (%s)", cfg.TlsMinVersion, cfg.TlsMaxVersion) + } + + // Check if user attempted to configure ciphers for TLS1.3 only: Go does not support that currently. + if minVersion == tls.VersionTLS13 && len(cfg.CipherSuites) > 0 { + return fmt.Errorf("cipher suites cannot be configured when only TLS1.3 is enabled") + } + + // Validate distributed tracing configuration but only if enabled. + if cfg.ExperimentalEnableDistributedTracing { + if err := validateTracingConfig(cfg.ExperimentalDistributedTracingSamplingRatePerMillion); err != nil { + return fmt.Errorf("distributed tracing configurition is not valid: (%v)", err) + } + } + return nil } @@ -687,7 +794,7 @@ func (cfg *Config) PeerURLsMapAndToken(which string) (urlsmap types.URLsMap, tok urlsmap = types.URLsMap{} // If using discovery, generate a temporary cluster based on // self's advertised peer URLs - urlsmap[cfg.Name] = cfg.APUrls + urlsmap[cfg.Name] = cfg.AdvertisePeerUrls token = cfg.Durl case cfg.DNSCluster != "": @@ -741,7 +848,7 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) { // Use both etcd-server-ssl and etcd-server for discovery. // Combine the results if both are available. - clusterStrs, cerr = getCluster("https", "etcd-server-ssl"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.APUrls) + clusterStrs, cerr = getCluster("https", "etcd-server-ssl"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.AdvertisePeerUrls) if cerr != nil { clusterStrs = make([]string, 0) } @@ -751,12 +858,12 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) { zap.String("service-name", "etcd-server-ssl"+serviceNameSuffix), zap.String("server-name", cfg.Name), zap.String("discovery-srv", cfg.DNSCluster), - zap.Strings("advertise-peer-urls", cfg.getAPURLs()), + zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerUrls()), zap.Strings("found-cluster", clusterStrs), zap.Error(cerr), ) - defaultHTTPClusterStrs, httpCerr := getCluster("http", "etcd-server"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.APUrls) + defaultHTTPClusterStrs, httpCerr := getCluster("http", "etcd-server"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.AdvertisePeerUrls) if httpCerr == nil { clusterStrs = append(clusterStrs, defaultHTTPClusterStrs...) } @@ -766,7 +873,7 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) { zap.String("service-name", "etcd-server"+serviceNameSuffix), zap.String("server-name", cfg.Name), zap.String("discovery-srv", cfg.DNSCluster), - zap.Strings("advertise-peer-urls", cfg.getAPURLs()), + zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerUrls()), zap.Strings("found-cluster", clusterStrs), zap.Error(httpCerr), ) @@ -775,15 +882,15 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) { } func (cfg Config) InitialClusterFromName(name string) (ret string) { - if len(cfg.APUrls) == 0 { + if len(cfg.AdvertisePeerUrls) == 0 { return "" } n := name if name == "" { n = DefaultName } - for i := range cfg.APUrls { - ret = ret + "," + n + "=" + cfg.APUrls[i].String() + for i := range cfg.AdvertisePeerUrls { + ret = ret + "," + n + "=" + cfg.AdvertisePeerUrls[i].String() } return ret[1:] } @@ -799,11 +906,11 @@ func (cfg Config) V2DeprecationEffective() config.V2DeprecationEnum { } func (cfg Config) defaultPeerHost() bool { - return len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs + return len(cfg.AdvertisePeerUrls) == 1 && cfg.AdvertisePeerUrls[0].String() == DefaultInitialAdvertisePeerURLs } func (cfg Config) defaultClientHost() bool { - return len(cfg.ACUrls) == 1 && cfg.ACUrls[0].String() == DefaultAdvertiseClientURLs + return len(cfg.AdvertiseClientUrls) == 1 && cfg.AdvertiseClientUrls[0].String() == DefaultAdvertiseClientURLs } func (cfg *Config) ClientSelfCert() (err error) { @@ -814,9 +921,12 @@ func (cfg *Config) ClientSelfCert() (err error) { cfg.logger.Warn("ignoring client auto TLS since certs given") return nil } - chosts := make([]string, len(cfg.LCUrls)) - for i, u := range cfg.LCUrls { - chosts[i] = u.Host + chosts := make([]string, 0, len(cfg.ListenClientUrls)+len(cfg.ListenClientHttpUrls)) + for _, u := range cfg.ListenClientUrls { + chosts = append(chosts, u.Host) + } + for _, u := range cfg.ListenClientHttpUrls { + chosts = append(chosts, u.Host) } cfg.ClientTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "client"), chosts, cfg.SelfSignedCertValidity) if err != nil { @@ -833,8 +943,8 @@ func (cfg *Config) PeerSelfCert() (err error) { cfg.logger.Warn("ignoring peer auto TLS since certs given") return nil } - phosts := make([]string, len(cfg.LPUrls)) - for i, u := range cfg.LPUrls { + phosts := make([]string, len(cfg.ListenPeerUrls)) + for i, u := range cfg.ListenPeerUrls { phosts[i] = u.Host } cfg.PeerTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "peer"), phosts, cfg.SelfSignedCertValidity) @@ -862,9 +972,9 @@ func (cfg *Config) UpdateDefaultClusterFromName(defaultInitialCluster string) (s } used := false - pip, pport := cfg.LPUrls[0].Hostname(), cfg.LPUrls[0].Port() + pip, pport := cfg.ListenPeerUrls[0].Hostname(), cfg.ListenPeerUrls[0].Port() if cfg.defaultPeerHost() && pip == "0.0.0.0" { - cfg.APUrls[0] = url.URL{Scheme: cfg.APUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, pport)} + cfg.AdvertisePeerUrls[0] = url.URL{Scheme: cfg.AdvertisePeerUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, pport)} used = true } // update 'initial-cluster' when only the name is specified (e.g. 'etcd --name=abc') @@ -872,9 +982,9 @@ func (cfg *Config) UpdateDefaultClusterFromName(defaultInitialCluster string) (s cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) } - cip, cport := cfg.LCUrls[0].Hostname(), cfg.LCUrls[0].Port() + cip, cport := cfg.ListenClientUrls[0].Hostname(), cfg.ListenClientUrls[0].Port() if cfg.defaultClientHost() && cip == "0.0.0.0" { - cfg.ACUrls[0] = url.URL{Scheme: cfg.ACUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, cport)} + cfg.AdvertiseClientUrls[0] = url.URL{Scheme: cfg.AdvertiseClientUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, cport)} used = true } dhost := defaultHostname @@ -919,34 +1029,42 @@ func checkHostURLs(urls []url.URL) error { return nil } -func (cfg *Config) getAPURLs() (ss []string) { - ss = make([]string, len(cfg.APUrls)) - for i := range cfg.APUrls { - ss[i] = cfg.APUrls[i].String() +func (cfg *Config) getAdvertisePeerUrls() (ss []string) { + ss = make([]string, len(cfg.AdvertisePeerUrls)) + for i := range cfg.AdvertisePeerUrls { + ss[i] = cfg.AdvertisePeerUrls[i].String() + } + return ss +} + +func (cfg *Config) getListenPeerUrls() (ss []string) { + ss = make([]string, len(cfg.ListenPeerUrls)) + for i := range cfg.ListenPeerUrls { + ss[i] = cfg.ListenPeerUrls[i].String() } return ss } -func (cfg *Config) getLPURLs() (ss []string) { - ss = make([]string, len(cfg.LPUrls)) - for i := range cfg.LPUrls { - ss[i] = cfg.LPUrls[i].String() +func (cfg *Config) getAdvertiseClientUrls() (ss []string) { + ss = make([]string, len(cfg.AdvertiseClientUrls)) + for i := range cfg.AdvertiseClientUrls { + ss[i] = cfg.AdvertiseClientUrls[i].String() } return ss } -func (cfg *Config) getACURLs() (ss []string) { - ss = make([]string, len(cfg.ACUrls)) - for i := range cfg.ACUrls { - ss[i] = cfg.ACUrls[i].String() +func (cfg *Config) getListenClientUrls() (ss []string) { + ss = make([]string, len(cfg.ListenClientUrls)) + for i := range cfg.ListenClientUrls { + ss[i] = cfg.ListenClientUrls[i].String() } return ss } -func (cfg *Config) getLCURLs() (ss []string) { - ss = make([]string, len(cfg.LCUrls)) - for i := range cfg.LCUrls { - ss[i] = cfg.LCUrls[i].String() +func (cfg *Config) getListenClientHttpUrls() (ss []string) { + ss = make([]string, len(cfg.ListenClientHttpUrls)) + for i := range cfg.ListenClientHttpUrls { + ss[i] = cfg.ListenClientHttpUrls[i].String() } return ss } diff --git a/server/embed/config_logging.go b/server/embed/config_logging.go index 645985f0f1df..1b363897b53c 100644 --- a/server/embed/config_logging.go +++ b/server/embed/config_logging.go @@ -87,7 +87,11 @@ func (cfg *Config) setupLogging() error { var path string if cfg.EnableLogRotation { // append rotate scheme to logs managed by lumberjack log rotation - path = fmt.Sprintf("rotate:%s", v) + if v[0:1] == "/" { + path = fmt.Sprintf("rotate:/%%2F%s", v[1:]) + } else { + path = fmt.Sprintf("rotate:/%s", v) + } } else { path = v } @@ -143,35 +147,38 @@ func (cfg *Config) setupLogging() error { return err } - logTLSHandshakeFailure := func(conn *tls.Conn, err error) { - state := conn.ConnectionState() - remoteAddr := conn.RemoteAddr().String() - serverName := state.ServerName - if len(state.PeerCertificates) > 0 { - cert := state.PeerCertificates[0] - ips := make([]string, len(cert.IPAddresses)) - for i := range cert.IPAddresses { - ips[i] = cert.IPAddresses[i].String() + logTLSHandshakeFailureFunc := func(msg string) func(conn *tls.Conn, err error) { + return func(conn *tls.Conn, err error) { + state := conn.ConnectionState() + remoteAddr := conn.RemoteAddr().String() + serverName := state.ServerName + if len(state.PeerCertificates) > 0 { + cert := state.PeerCertificates[0] + ips := make([]string, len(cert.IPAddresses)) + for i := range cert.IPAddresses { + ips[i] = cert.IPAddresses[i].String() + } + cfg.logger.Warn( + msg, + zap.String("remote-addr", remoteAddr), + zap.String("server-name", serverName), + zap.Strings("ip-addresses", ips), + zap.Strings("dns-names", cert.DNSNames), + zap.Error(err), + ) + } else { + cfg.logger.Warn( + msg, + zap.String("remote-addr", remoteAddr), + zap.String("server-name", serverName), + zap.Error(err), + ) } - cfg.logger.Warn( - "rejected connection", - zap.String("remote-addr", remoteAddr), - zap.String("server-name", serverName), - zap.Strings("ip-addresses", ips), - zap.Strings("dns-names", cert.DNSNames), - zap.Error(err), - ) - } else { - cfg.logger.Warn( - "rejected connection", - zap.String("remote-addr", remoteAddr), - zap.String("server-name", serverName), - zap.Error(err), - ) } } - cfg.ClientTLSInfo.HandshakeFailure = logTLSHandshakeFailure - cfg.PeerTLSInfo.HandshakeFailure = logTLSHandshakeFailure + + cfg.ClientTLSInfo.HandshakeFailure = logTLSHandshakeFailureFunc("rejected connection on client endpoint") + cfg.PeerTLSInfo.HandshakeFailure = logTLSHandshakeFailureFunc("rejected connection on peer endpoint") default: return fmt.Errorf("unknown logger option %q", cfg.Logger) @@ -254,7 +261,7 @@ func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { } } zap.RegisterSink("rotate", func(u *url.URL) (zap.Sink, error) { - logRotationConfig.Filename = u.Path + logRotationConfig.Filename = u.Path[1:] return &logRotationConfig, nil }) return nil diff --git a/server/embed/config_test.go b/server/embed/config_test.go index b86d0d6c2469..7307aa9c8fd0 100644 --- a/server/embed/config_test.go +++ b/server/embed/config_test.go @@ -15,6 +15,7 @@ package embed import ( + "crypto/tls" "errors" "fmt" "io/ioutil" @@ -24,6 +25,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" @@ -38,14 +40,16 @@ func notFoundErr(service, domain string) error { func TestConfigFileOtherFields(t *testing.T) { ctls := securityConfig{TrustedCAFile: "cca", CertFile: "ccert", KeyFile: "ckey"} - ptls := securityConfig{TrustedCAFile: "pca", CertFile: "pcert", KeyFile: "pkey"} + // Note AllowedCN and AllowedHostname are mutually exclusive, this test is just to verify the fields can be correctly marshalled & unmarshalled. + ptls := securityConfig{TrustedCAFile: "pca", CertFile: "pcert", KeyFile: "pkey", AllowedCNs: []string{"etcd"}, AllowedHostnames: []string{"whatever.example.com"}} yc := struct { - ClientSecurityCfgFile securityConfig `json:"client-transport-security"` - PeerSecurityCfgFile securityConfig `json:"peer-transport-security"` - ForceNewCluster bool `json:"force-new-cluster"` - Logger string `json:"logger"` - LogOutputs []string `json:"log-outputs"` - Debug bool `json:"debug"` + ClientSecurityCfgFile securityConfig `json:"client-transport-security"` + PeerSecurityCfgFile securityConfig `json:"peer-transport-security"` + ForceNewCluster bool `json:"force-new-cluster"` + Logger string `json:"logger"` + LogOutputs []string `json:"log-outputs"` + Debug bool `json:"debug"` + SocketOpts transport.SocketOpts `json:"socket-options"` }{ ctls, ptls, @@ -53,6 +57,9 @@ func TestConfigFileOtherFields(t *testing.T) { "zap", []string{"/dev/null"}, false, + transport.SocketOpts{ + ReusePort: true, + }, } b, err := yaml.Marshal(&yc) @@ -68,28 +75,30 @@ func TestConfigFileOtherFields(t *testing.T) { t.Fatal(err) } - if !cfg.ForceNewCluster { - t.Errorf("ForceNewCluster = %v, want %v", cfg.ForceNewCluster, true) - } - if !ctls.equals(&cfg.ClientTLSInfo) { t.Errorf("ClientTLS = %v, want %v", cfg.ClientTLSInfo, ctls) } if !ptls.equals(&cfg.PeerTLSInfo) { t.Errorf("PeerTLS = %v, want %v", cfg.PeerTLSInfo, ptls) } + + assert.Equal(t, true, cfg.ForceNewCluster, "ForceNewCluster does not match") + + assert.Equal(t, true, cfg.SocketOpts.ReusePort, "ReusePort does not match") + + assert.Equal(t, false, cfg.SocketOpts.ReuseAddress, "ReuseAddress does not match") } // TestUpdateDefaultClusterFromName ensures that etcd can start with 'etcd --name=abc'. func TestUpdateDefaultClusterFromName(t *testing.T) { cfg := NewConfig() defaultInitialCluster := cfg.InitialCluster - oldscheme := cfg.APUrls[0].Scheme - origpeer := cfg.APUrls[0].String() - origadvc := cfg.ACUrls[0].String() + oldscheme := cfg.AdvertisePeerUrls[0].Scheme + origpeer := cfg.AdvertisePeerUrls[0].String() + origadvc := cfg.AdvertiseClientUrls[0].String() cfg.Name = "abc" - lpport := cfg.LPUrls[0].Port() + lpport := cfg.ListenPeerUrls[0].Port() // in case of 'etcd --name=abc' exp := fmt.Sprintf("%s=%s://localhost:%s", cfg.Name, oldscheme, lpport) @@ -98,12 +107,12 @@ func TestUpdateDefaultClusterFromName(t *testing.T) { t.Fatalf("initial-cluster expected %q, got %q", exp, cfg.InitialCluster) } // advertise peer URL should not be affected - if origpeer != cfg.APUrls[0].String() { - t.Fatalf("advertise peer url expected %q, got %q", origadvc, cfg.APUrls[0].String()) + if origpeer != cfg.AdvertisePeerUrls[0].String() { + t.Fatalf("advertise peer url expected %q, got %q", origadvc, cfg.AdvertisePeerUrls[0].String()) } // advertise client URL should not be affected - if origadvc != cfg.ACUrls[0].String() { - t.Fatalf("advertise client url expected %q, got %q", origadvc, cfg.ACUrls[0].String()) + if origadvc != cfg.AdvertiseClientUrls[0].String() { + t.Fatalf("advertise client url expected %q, got %q", origadvc, cfg.AdvertiseClientUrls[0].String()) } } @@ -116,17 +125,17 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) { cfg := NewConfig() defaultInitialCluster := cfg.InitialCluster - oldscheme := cfg.APUrls[0].Scheme - origadvc := cfg.ACUrls[0].String() + oldscheme := cfg.AdvertisePeerUrls[0].Scheme + origadvc := cfg.AdvertiseClientUrls[0].String() cfg.Name = "abc" - lpport := cfg.LPUrls[0].Port() - cfg.LPUrls[0] = url.URL{Scheme: cfg.LPUrls[0].Scheme, Host: fmt.Sprintf("0.0.0.0:%s", lpport)} + lpport := cfg.ListenPeerUrls[0].Port() + cfg.ListenPeerUrls[0] = url.URL{Scheme: cfg.ListenPeerUrls[0].Scheme, Host: fmt.Sprintf("0.0.0.0:%s", lpport)} dhost, _ := cfg.UpdateDefaultClusterFromName(defaultInitialCluster) if dhost != defaultHostname { t.Fatalf("expected default host %q, got %q", defaultHostname, dhost) } - aphost, apport := cfg.APUrls[0].Hostname(), cfg.APUrls[0].Port() + aphost, apport := cfg.AdvertisePeerUrls[0].Hostname(), cfg.AdvertisePeerUrls[0].Port() if apport != lpport { t.Fatalf("advertise peer url got different port %s, expected %s", apport, lpport) } @@ -139,15 +148,32 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) { } // advertise client URL should not be affected - if origadvc != cfg.ACUrls[0].String() { - t.Fatalf("advertise-client-url expected %q, got %q", origadvc, cfg.ACUrls[0].String()) + if origadvc != cfg.AdvertiseClientUrls[0].String() { + t.Fatalf("advertise-client-url expected %q, got %q", origadvc, cfg.AdvertiseClientUrls[0].String()) } } func (s *securityConfig) equals(t *transport.TLSInfo) bool { return s.CertFile == t.CertFile && s.CertAuth == t.ClientCertAuth && - s.TrustedCAFile == t.TrustedCAFile + s.TrustedCAFile == t.TrustedCAFile && + s.ClientCertFile == t.ClientCertFile && + s.ClientKeyFile == t.ClientKeyFile && + s.KeyFile == t.KeyFile && + compareSlices(s.AllowedCNs, t.AllowedCNs) && + compareSlices(s.AllowedHostnames, t.AllowedHostnames) +} + +func compareSlices(slice1, slice2 []string) bool { + if len(slice1) != len(slice2) { + return false + } + for i, v := range slice1 { + if v != slice2[i] { + return false + } + } + return true } func mustCreateCfgFile(t *testing.T, b []byte) *os.File { @@ -274,7 +300,7 @@ func TestPeerURLsMapAndTokenFromSRV(t *testing.T) { cfg.InitialCluster = "" cfg.InitialClusterToken = "" cfg.DNSCluster = "example.com" - cfg.APUrls = types.MustNewURLs(tt.apurls) + cfg.AdvertisePeerUrls = types.MustNewURLs(tt.apurls) if err := cfg.Validate(); err != nil { t.Errorf("#%d: failed to validate test Config: %v", i, err) @@ -291,6 +317,56 @@ func TestPeerURLsMapAndTokenFromSRV(t *testing.T) { } } +func TestLeaseCheckpointValidate(t *testing.T) { + tcs := []struct { + name string + configFunc func() Config + expectError bool + }{ + { + name: "Default config should pass", + configFunc: func() Config { + return *NewConfig() + }, + }, + { + name: "Enabling checkpoint leases should pass", + configFunc: func() Config { + cfg := *NewConfig() + cfg.ExperimentalEnableLeaseCheckpoint = true + return cfg + }, + }, + { + name: "Enabling checkpoint leases and persist should pass", + configFunc: func() Config { + cfg := *NewConfig() + cfg.ExperimentalEnableLeaseCheckpoint = true + cfg.ExperimentalEnableLeaseCheckpointPersist = true + return cfg + }, + }, + { + name: "Enabling checkpoint leases persist without checkpointing itself should fail", + configFunc: func() Config { + cfg := *NewConfig() + cfg.ExperimentalEnableLeaseCheckpointPersist = true + return cfg + }, + expectError: true, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.configFunc() + err := cfg.Validate() + if (err != nil) != tc.expectError { + t.Errorf("config.Validate() = %q, expected error: %v", err, tc.expectError) + } + }) + } +} + func TestLogRotation(t *testing.T) { tests := []struct { name string @@ -304,6 +380,11 @@ func TestLogRotation(t *testing.T) { logOutputs: []string{"stderr", "/tmp/path"}, logRotationConfig: `{"maxsize": 1}`, }, + { + name: "log output relative path", + logOutputs: []string{"stderr", "tmp/path"}, + logRotationConfig: `{"maxsize": 1}`, + }, { name: "no file targets", logOutputs: []string{"stderr"}, @@ -361,6 +442,86 @@ func TestLogRotation(t *testing.T) { if err == nil && tt.wantErr { t.Errorf("test %q, expected error, got nil", tt.name) } + if err == nil { + cfg.GetLogger().Info("test log") + } + }) + } +} + +func TestTLSVersionMinMax(t *testing.T) { + tests := []struct { + name string + givenTLSMinVersion string + givenTLSMaxVersion string + givenCipherSuites []string + expectError bool + expectedMinTLSVersion uint16 + expectedMaxTLSVersion uint16 + }{ + { + name: "Minimum TLS version is set", + givenTLSMinVersion: "TLS1.3", + expectedMinTLSVersion: tls.VersionTLS13, + expectedMaxTLSVersion: 0, + }, + { + name: "Maximum TLS version is set", + givenTLSMaxVersion: "TLS1.2", + expectedMinTLSVersion: 0, + expectedMaxTLSVersion: tls.VersionTLS12, + }, + { + name: "Minimum and Maximum TLS versions are set", + givenTLSMinVersion: "TLS1.3", + givenTLSMaxVersion: "TLS1.3", + expectedMinTLSVersion: tls.VersionTLS13, + expectedMaxTLSVersion: tls.VersionTLS13, + }, + { + name: "Minimum and Maximum TLS versions are set in reverse order", + givenTLSMinVersion: "TLS1.3", + givenTLSMaxVersion: "TLS1.2", + expectError: true, + }, + { + name: "Invalid minimum TLS version", + givenTLSMinVersion: "invalid version", + expectError: true, + }, + { + name: "Invalid maximum TLS version", + givenTLSMaxVersion: "invalid version", + expectError: true, + }, + { + name: "Cipher suites configured for TLS 1.3", + givenTLSMinVersion: "TLS1.3", + givenCipherSuites: []string{"TLS_AES_128_GCM_SHA256"}, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := NewConfig() + cfg.TlsMinVersion = tt.givenTLSMinVersion + cfg.TlsMaxVersion = tt.givenTLSMaxVersion + cfg.CipherSuites = tt.givenCipherSuites + + err := cfg.Validate() + if err != nil { + assert.True(t, tt.expectError, "Validate() returned error while expecting success: %v", err) + return + } + + updateMinMaxVersions(&cfg.PeerTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) + updateMinMaxVersions(&cfg.ClientTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) + + assert.Equal(t, tt.expectedMinTLSVersion, cfg.PeerTLSInfo.MinVersion) + assert.Equal(t, tt.expectedMaxTLSVersion, cfg.PeerTLSInfo.MaxVersion) + assert.Equal(t, tt.expectedMinTLSVersion, cfg.ClientTLSInfo.MinVersion) + assert.Equal(t, tt.expectedMaxTLSVersion, cfg.ClientTLSInfo.MaxVersion) }) } } diff --git a/server/embed/config_tracing.go b/server/embed/config_tracing.go new file mode 100644 index 000000000000..382bc00e766b --- /dev/null +++ b/server/embed/config_tracing.go @@ -0,0 +1,138 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package embed + +import ( + "context" + "fmt" + + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "go.uber.org/zap" +) + +const maxSamplingRatePerMillion = 1000000 + +func validateTracingConfig(samplingRate int) error { + if samplingRate < 0 { + return fmt.Errorf("tracing sampling rate must be positive") + } + if samplingRate > maxSamplingRatePerMillion { + return fmt.Errorf("tracing sampling rate must be less than %d", maxSamplingRatePerMillion) + } + + return nil +} + +type tracingExporter struct { + exporter tracesdk.SpanExporter + opts []otelgrpc.Option + provider *tracesdk.TracerProvider +} + +func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, error) { + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(cfg.ExperimentalDistributedTracingAddress), + ) + if err != nil { + return nil, err + } + + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String(cfg.ExperimentalDistributedTracingServiceName), + ), + ) + if err != nil { + return nil, err + } + + if resWithIDKey := determineResourceWithIDKey(cfg.ExperimentalDistributedTracingServiceInstanceID); resWithIDKey != nil { + // Merge resources into a new + // resource in case of duplicates. + res, err = resource.Merge(res, resWithIDKey) + if err != nil { + return nil, err + } + } + + traceProvider := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exporter), + tracesdk.WithResource(res), + tracesdk.WithSampler( + tracesdk.ParentBased(determineSampler(cfg.ExperimentalDistributedTracingSamplingRatePerMillion)), + ), + ) + + options := []otelgrpc.Option{ + otelgrpc.WithPropagators( + propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ), + ), + otelgrpc.WithTracerProvider( + traceProvider, + ), + } + + cfg.logger.Debug( + "distributed tracing enabled", + zap.String("address", cfg.ExperimentalDistributedTracingAddress), + zap.String("service-name", cfg.ExperimentalDistributedTracingServiceName), + zap.String("service-instance-id", cfg.ExperimentalDistributedTracingServiceInstanceID), + ) + + return &tracingExporter{ + exporter: exporter, + opts: options, + provider: traceProvider, + }, nil +} + +func (te *tracingExporter) Close(ctx context.Context) { + if te.provider != nil { + te.provider.Shutdown(ctx) + } + + if te.exporter != nil { + te.exporter.Shutdown(ctx) + } +} + +func determineSampler(samplingRate int) tracesdk.Sampler { + sampler := tracesdk.NeverSample() + if samplingRate == 0 { + return sampler + } + return tracesdk.TraceIDRatioBased(float64(samplingRate) / float64(maxSamplingRatePerMillion)) +} + +// As Tracing service Instance ID must be unique, it should +// never use the empty default string value, it's set if +// if it's a non empty string. +func determineResourceWithIDKey(serviceInstanceID string) *resource.Resource { + if serviceInstanceID != "" { + return resource.NewSchemaless( + (semconv.ServiceInstanceIDKey.String(serviceInstanceID)), + ) + } + return nil +} diff --git a/server/embed/config_tracing_test.go b/server/embed/config_tracing_test.go new file mode 100644 index 000000000000..0abbe4d1d42d --- /dev/null +++ b/server/embed/config_tracing_test.go @@ -0,0 +1,83 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package embed + +import ( + "testing" +) + +const neverSampleDescription = "AlwaysOffSampler" + +func TestDetermineSampler(t *testing.T) { + tests := []struct { + name string + sampleRate int + wantSamplerDescription string + }{ + { + name: "sample rate is disabled", + sampleRate: 0, + wantSamplerDescription: neverSampleDescription, + }, + { + name: "sample rate is 100", + sampleRate: 100, + wantSamplerDescription: "TraceIDRatioBased{0.0001}", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + sampler := determineSampler(tc.sampleRate) + if tc.wantSamplerDescription != sampler.Description() { + t.Errorf("tracing sampler was not as expected; expected sampler: %#+v, got sampler: %#+v", tc.wantSamplerDescription, sampler.Description()) + } + }) + } +} + +func TestTracingConfig(t *testing.T) { + tests := []struct { + name string + sampleRate int + wantErr bool + }{ + { + name: "invalid - sample rate is less than 0", + sampleRate: -1, + wantErr: true, + }, + { + name: "invalid - sample rate is more than allowed value", + sampleRate: maxSamplingRatePerMillion + 1, + wantErr: true, + }, + { + name: "valid - sample rate is 100", + sampleRate: 100, + wantErr: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validateTracingConfig(tc.sampleRate) + if err == nil && tc.wantErr { + t.Errorf("expected error got (%v) error", err) + } + if err != nil && !tc.wantErr { + t.Errorf("expected no errors, got error: (%v)", err) + } + }) + } +} diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 001302f991b7..30cbff3d22ad 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -17,9 +17,11 @@ package embed import ( "context" "crypto/tls" + "errors" "fmt" "io/ioutil" defaultLog "log" + "math" "net" "net/http" "net/url" @@ -32,6 +34,7 @@ import ( "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/client/v3/credentials" "go.etcd.io/etcd/pkg/v3/debugutil" runtimeutil "go.etcd.io/etcd/pkg/v3/runtime" "go.etcd.io/etcd/server/v3/config" @@ -46,15 +49,9 @@ import ( grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/soheilhy/cmux" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel/exporters/otlp" - "go.opentelemetry.io/otel/exporters/otlp/otlpgrpc" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - tracesdk "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/semconv" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" ) @@ -84,11 +81,23 @@ type Etcd struct { Server *etcdserver.EtcdServer - cfg Config - stopc chan struct{} - errc chan error + cfg Config + // closeOnce is to ensure `stopc` is closed only once, no matter + // how many times the Close() method is called. closeOnce sync.Once + // stopc is used to notify the sub goroutines not to send + // any errors to `errc`. + stopc chan struct{} + // errc is used to receive error from sub goroutines (including + // client handler, peer handler and metrics handler). It's closed + // after all these sub goroutines exit (checked via `wg`). Writers + // should avoid writing after `stopc` is closed by selecting on + // reading from `stopc`. + errc chan error + + // wg is used to track the lifecycle of all sub goroutines created by `StartEtcd`. + wg sync.WaitGroup } type peerListener struct { @@ -114,7 +123,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { if !serving { // errored before starting gRPC server for serveCtx.serversC for _, sctx := range e.sctxs { - close(sctx.serversC) + sctx.close() } } e.Close() @@ -130,7 +139,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { } e.cfg.logger.Info( "configuring peer listeners", - zap.Strings("listen-peer-urls", e.cfg.getLPURLs()), + zap.Strings("listen-peer-urls", e.cfg.getListenPeerUrls()), ) if e.Peers, err = configurePeerListeners(cfg); err != nil { return e, err @@ -138,7 +147,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { e.cfg.logger.Info( "configuring client listeners", - zap.Strings("listen-client-urls", e.cfg.getLCURLs()), + zap.Strings("listen-client-urls", e.cfg.getListenClientUrls()), ) if e.sctxs, err = configureClientListeners(cfg); err != nil { return e, err @@ -174,8 +183,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { srvcfg := config.ServerConfig{ Name: cfg.Name, - ClientURLs: cfg.ACUrls, - PeerURLs: cfg.APUrls, + ClientURLs: cfg.AdvertiseClientUrls, + PeerURLs: cfg.AdvertisePeerUrls, DataDir: cfg.Dir, DedicatedWALDir: cfg.WalDir, SnapshotCount: cfg.SnapshotCount, @@ -199,6 +208,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { BackendBatchInterval: cfg.BackendBatchInterval, MaxTxnOps: cfg.MaxTxnOps, MaxRequestBytes: cfg.MaxRequestBytes, + MaxConcurrentStreams: cfg.MaxConcurrentStreams, SocketOpts: cfg.SocketOpts, StrictReconfigCheck: cfg.StrictReconfigCheck, ClientCertAuthEnabled: cfg.ClientTLSInfo.ClientCertAuth, @@ -209,34 +219,42 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { HostWhitelist: cfg.HostWhitelist, InitialCorruptCheck: cfg.ExperimentalInitialCorruptCheck, CorruptCheckTime: cfg.ExperimentalCorruptCheckTime, + CompactHashCheckEnabled: cfg.ExperimentalCompactHashCheckEnabled, + CompactHashCheckTime: cfg.ExperimentalCompactHashCheckTime, PreVote: cfg.PreVote, Logger: cfg.logger, ForceNewCluster: cfg.ForceNewCluster, + ForceNewClusterBumpAmount: cfg.ForceNewClusterBumpAmount, EnableGRPCGateway: cfg.EnableGRPCGateway, ExperimentalEnableDistributedTracing: cfg.ExperimentalEnableDistributedTracing, UnsafeNoFsync: cfg.UnsafeNoFsync, EnableLeaseCheckpoint: cfg.ExperimentalEnableLeaseCheckpoint, + LeaseCheckpointPersist: cfg.ExperimentalEnableLeaseCheckpointPersist, CompactionBatchLimit: cfg.ExperimentalCompactionBatchLimit, + CompactionSleepInterval: cfg.ExperimentalCompactionSleepInterval, WatchProgressNotifyInterval: cfg.ExperimentalWatchProgressNotifyInterval, DowngradeCheckTime: cfg.ExperimentalDowngradeCheckTime, WarningApplyDuration: cfg.ExperimentalWarningApplyDuration, ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer, + ExperimentalStopGRPCServiceOnDefrag: cfg.ExperimentalStopGRPCServiceOnDefrag, ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, - V2Deprecation: cfg.V2DeprecationEffective(), + ExperimentalMaxLearners: cfg.ExperimentalMaxLearners, + V2Deprecation: cfg.V2DeprecationEffective(), } if srvcfg.ExperimentalEnableDistributedTracing { tctx := context.Background() - tracingExporter, opts, err := e.setupTracing(tctx) + tracingExporter, err := newTracingExporter(tctx, cfg) if err != nil { return e, err } - if tracingExporter == nil || len(opts) == 0 { - return e, fmt.Errorf("error setting up distributed tracing") + e.tracingExporterShutdown = func() { + tracingExporter.Close(tctx) } - e.tracingExporterShutdown = func() { tracingExporter.Shutdown(tctx) } - srvcfg.ExperimentalTracerOptions = opts + srvcfg.ExperimentalTracerOptions = tracingExporter.opts + + e.cfg.logger.Info("distributed tracing setup enabled") } print(e.cfg.logger, *cfg, srvcfg, memberInitialized) @@ -250,8 +268,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { // newly started member ("memberInitialized==false") // does not need corruption check - if memberInitialized { - if err = e.Server.CheckInitialHashKV(); err != nil { + if memberInitialized && srvcfg.InitialCorruptCheck { + if err = e.Server.CorruptionChecker().InitialCheck(); err != nil { // set "EtcdServer" to nil, so that it does not block on "EtcdServer.Close()" // (nothing to close since rafthttp transports have not been started) @@ -276,10 +294,10 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { e.cfg.logger.Info( "now serving peer/client/metrics", zap.String("local-member-id", e.Server.ID().String()), - zap.Strings("initial-advertise-peer-urls", e.cfg.getAPURLs()), - zap.Strings("listen-peer-urls", e.cfg.getLPURLs()), - zap.Strings("advertise-client-urls", e.cfg.getACURLs()), - zap.Strings("listen-client-urls", e.cfg.getLCURLs()), + zap.Strings("initial-advertise-peer-urls", e.cfg.getAdvertisePeerUrls()), + zap.Strings("listen-peer-urls", e.cfg.getListenPeerUrls()), + zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientUrls()), + zap.Strings("listen-client-urls", e.cfg.getListenClientUrls()), zap.Strings("listen-metrics-urls", e.cfg.getMetricsURLs()), ) serving = true @@ -324,27 +342,35 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized zap.String("election-timeout", fmt.Sprintf("%v", time.Duration(sc.ElectionTicks*int(sc.TickMs))*time.Millisecond)), zap.Bool("initial-election-tick-advance", sc.InitialElectionTickAdvance), zap.Uint64("snapshot-count", sc.SnapshotCount), + zap.Uint("max-wals", sc.MaxWALFiles), + zap.Uint("max-snapshots", sc.MaxSnapFiles), zap.Uint64("snapshot-catchup-entries", sc.SnapshotCatchUpEntries), - zap.Strings("initial-advertise-peer-urls", ec.getAPURLs()), - zap.Strings("listen-peer-urls", ec.getLPURLs()), - zap.Strings("advertise-client-urls", ec.getACURLs()), - zap.Strings("listen-client-urls", ec.getLCURLs()), + zap.Strings("initial-advertise-peer-urls", ec.getAdvertisePeerUrls()), + zap.Strings("listen-peer-urls", ec.getListenPeerUrls()), + zap.Strings("advertise-client-urls", ec.getAdvertiseClientUrls()), + zap.Strings("listen-client-urls", ec.getListenClientUrls()), zap.Strings("listen-metrics-urls", ec.getMetricsURLs()), zap.Strings("cors", cors), zap.Strings("host-whitelist", hss), zap.String("initial-cluster", sc.InitialPeerURLsMap.String()), zap.String("initial-cluster-state", ec.ClusterState), zap.String("initial-cluster-token", sc.InitialClusterToken), - zap.Int64("quota-size-bytes", quota), + zap.Int64("quota-backend-bytes", quota), + zap.Uint("max-request-bytes", sc.MaxRequestBytes), + zap.Uint32("max-concurrent-streams", sc.MaxConcurrentStreams), + zap.Bool("pre-vote", sc.PreVote), zap.Bool("initial-corrupt-check", sc.InitialCorruptCheck), zap.String("corrupt-check-time-interval", sc.CorruptCheckTime.String()), + zap.Bool("compact-check-time-enabled", sc.CompactHashCheckEnabled), + zap.Duration("compact-check-time-interval", sc.CompactHashCheckTime), zap.String("auto-compaction-mode", sc.AutoCompactionMode), zap.Duration("auto-compaction-retention", sc.AutoCompactionRetention), zap.String("auto-compaction-interval", sc.AutoCompactionRetention.String()), zap.String("discovery-url", sc.DiscoveryURL), zap.String("discovery-proxy", sc.DiscoveryProxy), zap.String("downgrade-check-interval", sc.DowngradeCheckTime.String()), + zap.Int("max-learners", sc.ExperimentalMaxLearners), ) } @@ -356,12 +382,30 @@ func (e *Etcd) Config() Config { // Close gracefully shuts down all servers/listeners. // Client requests will be terminated with request timeout. // After timeout, enforce remaning requests be closed immediately. +// +// The rough workflow to shut down etcd: +// 1. close the `stopc` channel, so that all error handlers (child +// goroutines) won't send back any errors anymore; +// 2. stop the http and grpc servers gracefully, within request timeout; +// 3. close all client and metrics listeners, so that etcd server +// stops receiving any new connection; +// 4. call the cancel function to close the gateway context, so that +// all gateway connections are closed. +// 5. stop etcd server gracefully, and ensure the main raft loop +// goroutine is stopped; +// 6. stop all peer listeners, so that it stops receiving peer connections +// and messages (wait up to 1-second); +// 7. wait for all child goroutines (i.e. client handlers, peer handlers +// and metrics handlers) to exit; +// 8. close the `errc` channel to release the resource. Note that it's only +// safe to close the `errc` after step 7 above is done, otherwise the +// child goroutines may send errors back to already closed `errc` channel. func (e *Etcd) Close() { fields := []zap.Field{ zap.String("name", e.cfg.Name), zap.String("data-dir", e.cfg.Dir), - zap.Strings("advertise-peer-urls", e.cfg.getAPURLs()), - zap.Strings("advertise-client-urls", e.cfg.getACURLs()), + zap.Strings("advertise-peer-urls", e.cfg.getAdvertisePeerUrls()), + zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientUrls()), } lg := e.GetLogger() lg.Info("closing etcd server", fields...) @@ -425,17 +469,23 @@ func (e *Etcd) Close() { } } if e.errc != nil { + e.wg.Wait() close(e.errc) } } func stopServers(ctx context.Context, ss *servers) { // first, close the http.Server - ss.http.Shutdown(ctx) - // do not grpc.Server.GracefulStop with TLS enabled etcd server + if ss.http != nil { + ss.http.Shutdown(ctx) + } + if ss.grpc == nil { + return + } + // do not grpc.Server.GracefulStop when grpc runs under http server // See https://github.com/grpc/grpc-go/issues/1384#issuecomment-317124531 // and https://github.com/etcd-io/etcd/issues/8916 - if ss.secure { + if ss.secure && ss.http != nil { ss.grpc.Stop() return } @@ -474,6 +524,9 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) { if err = cfg.PeerSelfCert(); err != nil { cfg.logger.Fatal("failed to get peer self-signed certs", zap.Error(err)) } + + updateMinMaxVersions(&cfg.PeerTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) + if !cfg.PeerTLSInfo.Empty() { cfg.logger.Info( "starting with peer TLS", @@ -482,7 +535,7 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) { ) } - peers = make([]*peerListener, len(cfg.LPUrls)) + peers = make([]*peerListener, len(cfg.ListenPeerUrls)) defer func() { if err == nil { return @@ -491,7 +544,7 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) { if peers[i] != nil && peers[i].close != nil { cfg.logger.Warn( "closing peer listener", - zap.String("address", cfg.LPUrls[i].String()), + zap.String("address", cfg.ListenPeerUrls[i].String()), zap.Error(err), ) ctx, cancel := context.WithTimeout(context.Background(), time.Second) @@ -501,7 +554,7 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) { } }() - for i, u := range cfg.LPUrls { + for i, u := range cfg.ListenPeerUrls { if u.Scheme == "http" { if !cfg.PeerTLSInfo.Empty() { cfg.logger.Warn("scheme is HTTP while key and cert files are present; ignoring key and cert files", zap.String("peer-url", u.String())) @@ -517,6 +570,7 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) { transport.WithTimeout(rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout), ) if err != nil { + cfg.logger.Error("creating peer listener failed", zap.Error(err)) return nil, err } // once serve, overwrite with 'http.Server.Shutdown' @@ -539,7 +593,7 @@ func (e *Etcd) servePeers() (err error) { for _, p := range e.Peers { u := p.Listener.Addr().String() - gs := v3rpc.Server(e.Server, peerTLScfg) + gs := v3rpc.Server(e.Server, peerTLScfg, nil) m := cmux.New(p.Listener) go gs.Serve(m.Match(cmux.HTTP2())) srv := &http.Server{ @@ -575,7 +629,9 @@ func (e *Etcd) servePeers() (err error) { // start peer servers in a goroutine for _, pl := range e.Peers { + e.wg.Add(1) go func(l *peerListener) { + defer e.wg.Done() u := l.Addr().String() e.cfg.logger.Info( "serving peer traffic", @@ -594,42 +650,61 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro if err = cfg.ClientSelfCert(); err != nil { cfg.logger.Fatal("failed to get client self-signed certs", zap.Error(err)) } + + updateMinMaxVersions(&cfg.ClientTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) + if cfg.EnablePprof { cfg.logger.Info("pprof is enabled", zap.String("path", debugutil.HTTPPrefixPProf)) } sctxs = make(map[string]*serveCtx) - for _, u := range cfg.LCUrls { - sctx := newServeCtx(cfg.logger) + for _, u := range append(cfg.ListenClientUrls, cfg.ListenClientHttpUrls...) { if u.Scheme == "http" || u.Scheme == "unix" { if !cfg.ClientTLSInfo.Empty() { - cfg.logger.Warn("scheme is HTTP while key and cert files are present; ignoring key and cert files", zap.String("client-url", u.String())) + cfg.logger.Warn("scheme is http or unix while key and cert files are present; ignoring key and cert files", zap.String("client-url", u.String())) } if cfg.ClientTLSInfo.ClientCertAuth { - cfg.logger.Warn("scheme is HTTP while --client-cert-auth is enabled; ignoring client cert auth for this URL", zap.String("client-url", u.String())) + cfg.logger.Warn("scheme is http or unix while --client-cert-auth is enabled; ignoring client cert auth for this URL", zap.String("client-url", u.String())) } } if (u.Scheme == "https" || u.Scheme == "unixs") && cfg.ClientTLSInfo.Empty() { return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPS scheme", u.String()) } + } - network := "tcp" - addr := u.Host - if u.Scheme == "unix" || u.Scheme == "unixs" { - network = "unix" - addr = u.Host + u.Path + for _, u := range cfg.ListenClientUrls { + addr, secure, network := resolveUrl(u) + sctx := sctxs[addr] + if sctx == nil { + sctx = newServeCtx(cfg.logger) + sctxs[addr] = sctx } + sctx.secure = sctx.secure || secure + sctx.insecure = sctx.insecure || !secure + sctx.scheme = u.Scheme + sctx.addr = addr sctx.network = network + } + for _, u := range cfg.ListenClientHttpUrls { + addr, secure, network := resolveUrl(u) - sctx.secure = u.Scheme == "https" || u.Scheme == "unixs" - sctx.insecure = !sctx.secure - if oldctx := sctxs[addr]; oldctx != nil { - oldctx.secure = oldctx.secure || sctx.secure - oldctx.insecure = oldctx.insecure || sctx.insecure - continue + sctx := sctxs[addr] + if sctx == nil { + sctx = newServeCtx(cfg.logger) + sctxs[addr] = sctx + } else if !sctx.httpOnly { + return nil, fmt.Errorf("cannot bind both --client-listen-urls and --client-listen-http-urls on the same url %s", u.String()) } + sctx.secure = sctx.secure || secure + sctx.insecure = sctx.insecure || !secure + sctx.scheme = u.Scheme + sctx.addr = addr + sctx.network = network + sctx.httpOnly = true + } - if sctx.l, err = transport.NewListenerWithOpts(addr, u.Scheme, + for _, sctx := range sctxs { + if sctx.l, err = transport.NewListenerWithOpts(sctx.addr, sctx.scheme, transport.WithSocketOpts(&cfg.SocketOpts), transport.WithSkipTLSInfoCheck(true), ); err != nil { @@ -637,7 +712,6 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro } // net.Listener will rewrite ipv4 0.0.0.0 to ipv6 [::], breaking // hosts that disable ipv6. So, use the address given by the user. - sctx.addr = addr if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil { if fdLimit <= reservedInternalFDNum { @@ -650,23 +724,17 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum)) } - if network == "tcp" { - if sctx.l, err = transport.NewKeepAliveListener(sctx.l, network, nil); err != nil { - return nil, err - } - } - - defer func(u url.URL) { - if err == nil { + defer func(sctx *serveCtx) { + if err == nil || sctx.l == nil { return } sctx.l.Close() cfg.logger.Warn( "closing peer listener", - zap.String("address", u.Host), + zap.String("address", sctx.addr), zap.Error(err), ) - }(u) + }(sctx) for k := range cfg.UserHandlers { sctx.userHandlers[k] = cfg.UserHandlers[k] } @@ -677,11 +745,21 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro if cfg.LogLevel == "debug" { sctx.registerTrace() } - sctxs[addr] = sctx } return sctxs, nil } +func resolveUrl(u url.URL) (addr string, secure bool, network string) { + addr = u.Host + network = "tcp" + if u.Scheme == "unix" || u.Scheme == "unixs" { + addr = u.Host + u.Path + network = "unix" + } + secure = u.Scheme == "https" || u.Scheme == "unixs" + return addr, secure, network +} + func (e *Etcd) serveClients() (err error) { if !e.cfg.ClientTLSInfo.Empty() { e.cfg.logger.Info( @@ -708,7 +786,8 @@ func (e *Etcd) serveClients() (err error) { } else { mux := http.NewServeMux() etcdhttp.HandleBasic(e.cfg.logger, mux, e.Server) - etcdhttp.HandleMetricsHealthForV3(e.cfg.logger, mux, e.Server) + etcdhttp.HandleMetrics(mux) + etcdhttp.HandleHealth(e.cfg.logger, mux, e.Server) h = mux } @@ -727,15 +806,89 @@ func (e *Etcd) serveClients() (err error) { })) } + splitHttp := false + for _, sctx := range e.sctxs { + if sctx.httpOnly { + splitHttp = true + } + } + // start client servers in each goroutine for _, sctx := range e.sctxs { + e.wg.Add(1) go func(s *serveCtx) { - e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler, gopts...)) + defer e.wg.Done() + e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler, e.grpcGatewayDial(splitHttp), splitHttp, gopts...)) }(sctx) } return nil } +func (e *Etcd) grpcGatewayDial(splitHttp bool) (grpcDial func(ctx context.Context) (*grpc.ClientConn, error)) { + if !e.cfg.EnableGRPCGateway { + return nil + } + sctx := e.pickGrpcGatewayServeContext(splitHttp) + addr := sctx.addr + if network := sctx.network; network == "unix" { + // explicitly define unix network for gRPC socket support + addr = fmt.Sprintf("%s:%s", network, addr) + } + + opts := []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32))} + if sctx.secure { + tlscfg, tlsErr := e.cfg.ClientTLSInfo.ServerConfig() + if tlsErr != nil { + return func(ctx context.Context) (*grpc.ClientConn, error) { + return nil, tlsErr + } + } + dtls := tlscfg.Clone() + // trust local server + dtls.InsecureSkipVerify = true + bundle := credentials.NewBundle(credentials.Config{TLSConfig: dtls}) + opts = append(opts, grpc.WithTransportCredentials(bundle.TransportCredentials())) + } else { + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + return func(ctx context.Context) (*grpc.ClientConn, error) { + conn, err := grpc.DialContext(ctx, addr, opts...) + if err != nil { + sctx.lg.Error("grpc gateway failed to dial", zap.String("addr", addr), zap.Error(err)) + return nil, err + } + return conn, err + } +} + +func (e *Etcd) pickGrpcGatewayServeContext(splitHttp bool) *serveCtx { + for _, sctx := range e.sctxs { + if !splitHttp || !sctx.httpOnly { + return sctx + } + } + panic("Expect at least one context able to serve grpc") +} + +var ErrMissingClientTLSInfoForMetricsURL = errors.New("client TLS key/cert (--cert-file, --key-file) must be provided for metrics secure url") + +func (e *Etcd) createMetricsListener(murl url.URL) (net.Listener, error) { + tlsInfo := &e.cfg.ClientTLSInfo + switch murl.Scheme { + case "http": + tlsInfo = nil + case "https", "unixs": + if e.cfg.ClientTLSInfo.Empty() { + return nil, ErrMissingClientTLSInfoForMetricsURL + } + } + return transport.NewListenerWithOpts(murl.Host, murl.Scheme, + transport.WithTLSInfo(tlsInfo), + transport.WithSocketOpts(&e.cfg.SocketOpts), + ) +} + func (e *Etcd) serveMetrics() (err error) { if e.cfg.Metrics == "extensive" { grpc_prometheus.EnableHandlingTimeHistogram() @@ -743,22 +896,18 @@ func (e *Etcd) serveMetrics() (err error) { if len(e.cfg.ListenMetricsUrls) > 0 { metricsMux := http.NewServeMux() - etcdhttp.HandleMetricsHealthForV3(e.cfg.logger, metricsMux, e.Server) + etcdhttp.HandleMetrics(metricsMux) + etcdhttp.HandleHealth(e.cfg.logger, metricsMux, e.Server) for _, murl := range e.cfg.ListenMetricsUrls { - tlsInfo := &e.cfg.ClientTLSInfo - if murl.Scheme == "http" { - tlsInfo = nil - } - ml, err := transport.NewListenerWithOpts(murl.Host, murl.Scheme, - transport.WithTLSInfo(tlsInfo), - transport.WithSocketOpts(&e.cfg.SocketOpts), - ) + ml, err := e.createMetricsListener(murl) if err != nil { return err } e.metricsListeners = append(e.metricsListeners, ml) + e.wg.Add(1) go func(u url.URL, ln net.Listener) { + defer e.wg.Done() e.cfg.logger.Info( "serving metrics", zap.String("address", u.String()), @@ -808,52 +957,3 @@ func parseCompactionRetention(mode, retention string) (ret time.Duration, err er } return ret, nil } - -func (e *Etcd) setupTracing(ctx context.Context) (exporter tracesdk.SpanExporter, options []otelgrpc.Option, err error) { - exporter, err = otlp.NewExporter(ctx, - otlpgrpc.NewDriver( - otlpgrpc.WithEndpoint(e.cfg.ExperimentalDistributedTracingAddress), - otlpgrpc.WithInsecure(), - )) - if err != nil { - return nil, nil, err - } - res := resource.NewWithAttributes( - semconv.ServiceNameKey.String(e.cfg.ExperimentalDistributedTracingServiceName), - ) - // As Tracing service Instance ID must be unique, it should - // never use the empty default string value, so we only set it - // if it's a non empty string. - if e.cfg.ExperimentalDistributedTracingServiceInstanceID != "" { - resWithIDKey := resource.NewWithAttributes( - (semconv.ServiceInstanceIDKey.String(e.cfg.ExperimentalDistributedTracingServiceInstanceID)), - ) - // Merge resources to combine into a new - // resource in case of duplicates. - res = resource.Merge(res, resWithIDKey) - } - - options = append(options, - otelgrpc.WithPropagators( - propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - ), - ), - otelgrpc.WithTracerProvider( - tracesdk.NewTracerProvider( - tracesdk.WithBatcher(exporter), - tracesdk.WithResource(res), - ), - ), - ) - - e.cfg.logger.Info( - "distributed tracing enabled", - zap.String("distributed-tracing-address", e.cfg.ExperimentalDistributedTracingAddress), - zap.String("distributed-tracing-service-name", e.cfg.ExperimentalDistributedTracingServiceName), - zap.String("distributed-tracing-service-instance-id", e.cfg.ExperimentalDistributedTracingServiceInstanceID), - ) - - return exporter, options, err -} diff --git a/server/embed/etcd_test.go b/server/embed/etcd_test.go new file mode 100644 index 000000000000..e22b6d3544a2 --- /dev/null +++ b/server/embed/etcd_test.go @@ -0,0 +1,38 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package embed + +import ( + "net/url" + "testing" + + "go.etcd.io/etcd/client/pkg/v3/transport" +) + +func TestEmptyClientTLSInfo_createMetricsListener(t *testing.T) { + e := &Etcd{ + cfg: Config{ + ClientTLSInfo: transport.TLSInfo{}, + }, + } + + murl := url.URL{ + Scheme: "https", + Host: "localhost:8080", + } + if _, err := e.createMetricsListener(murl); err != ErrMissingClientTLSInfoForMetricsURL { + t.Fatalf("expected error %v, got %v", ErrMissingClientTLSInfoForMetricsURL, err) + } +} diff --git a/server/embed/serve.go b/server/embed/serve.go index 23e115d6a5e9..55ffe3743f4e 100644 --- a/server/embed/serve.go +++ b/server/embed/serve.go @@ -16,18 +16,20 @@ package embed import ( "context" + "errors" "fmt" "io/ioutil" defaultLog "log" "net" "net/http" "strings" + "sync" etcdservergw "go.etcd.io/etcd/api/v3/etcdserverpb/gw" "go.etcd.io/etcd/client/pkg/v3/transport" - "go.etcd.io/etcd/client/v3/credentials" "go.etcd.io/etcd/pkg/v3/debugutil" "go.etcd.io/etcd/pkg/v3/httputil" + "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election" @@ -42,24 +44,39 @@ import ( "github.com/soheilhy/cmux" "github.com/tmc/grpc-websocket-proxy/wsproxy" "go.uber.org/zap" + "golang.org/x/net/http2" "golang.org/x/net/trace" "google.golang.org/grpc" ) type serveCtx struct { - lg *zap.Logger - l net.Listener + lg *zap.Logger + l net.Listener + + scheme string addr string network string secure bool insecure bool + httpOnly bool + // ctx is used to control the grpc gateway. Terminate the grpc gateway + // by calling `cancel` when shutting down the etcd. ctx context.Context cancel context.CancelFunc userHandlers map[string]http.Handler serviceRegister func(*grpc.Server) - serversC chan *servers + + // serversC is used to receive the http and grpc server objects (created + // in `serve`), both of which will be closed when shutting down the etcd. + // Close it when `serve` returns or when etcd fails to bootstrap. + serversC chan *servers + // closeOnce is to ensure `serversC` is closed only once. + closeOnce sync.Once + + // wg is used to track the lifecycle of all sub goroutines created by `serve`. + wg sync.WaitGroup } type servers struct { @@ -90,108 +107,190 @@ func (sctx *serveCtx) serve( tlsinfo *transport.TLSInfo, handler http.Handler, errHandler func(error), + grpcDialForRestGatewayBackends func(ctx context.Context) (*grpc.ClientConn, error), + splitHttp bool, gopts ...grpc.ServerOption) (err error) { logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) - <-s.ReadyNotify() + + // Make sure serversC is closed even if we prematurely exit the function. + defer sctx.close() + + select { + case <-s.StoppingNotify(): + return errors.New("server is stopping") + case <-s.ReadyNotify(): + } sctx.lg.Info("ready to serve client requests") m := cmux.New(sctx.l) + var server func() error + onlyGRPC := splitHttp && !sctx.httpOnly + onlyHttp := splitHttp && sctx.httpOnly + grpcEnabled := !onlyHttp + httpEnabled := !onlyGRPC + v3c := v3client.New(s) servElection := v3election.NewElectionServer(v3c) servLock := v3lock.NewLockServer(v3c) - var gs *grpc.Server - defer func() { - if err != nil && gs != nil { - gs.Stop() + var gwmux *gw.ServeMux + if s.Cfg.EnableGRPCGateway { + // GRPC gateway connects to grpc server via connection provided by grpc dial. + gwmux, err = sctx.registerGateway(grpcDialForRestGatewayBackends) + if err != nil { + sctx.lg.Error("registerGateway failed", zap.Error(err)) + return err } - }() + } + var traffic string + switch { + case onlyGRPC: + traffic = "grpc" + case onlyHttp: + traffic = "http" + default: + traffic = "grpc+http" + } if sctx.insecure { - gs = v3rpc.Server(s, nil, gopts...) - v3electionpb.RegisterElectionServer(gs, servElection) - v3lockpb.RegisterLockServer(gs, servLock) - if sctx.serviceRegister != nil { - sctx.serviceRegister(gs) - } - grpcl := m.Match(cmux.HTTP2()) - go func() { errHandler(gs.Serve(grpcl)) }() - - var gwmux *gw.ServeMux - if s.Cfg.EnableGRPCGateway { - gwmux, err = sctx.registerGateway([]grpc.DialOption{grpc.WithInsecure()}) - if err != nil { + var gs *grpc.Server + var srv *http.Server + if httpEnabled { + httpmux := sctx.createMux(gwmux, handler) + srv = &http.Server{ + Handler: createAccessController(sctx.lg, s, httpmux), + ErrorLog: logger, // do not log user error + } + if err := configureHttpServer(srv, s.Cfg); err != nil { + sctx.lg.Error("Configure http server failed", zap.Error(err)) return err } } - - httpmux := sctx.createMux(gwmux, handler) - - srvhttp := &http.Server{ - Handler: createAccessController(sctx.lg, s, httpmux), - ErrorLog: logger, // do not log user error + if grpcEnabled { + gs = v3rpc.Server(s, nil, nil, gopts...) + v3electionpb.RegisterElectionServer(gs, servElection) + v3lockpb.RegisterLockServer(gs, servLock) + if sctx.serviceRegister != nil { + sctx.serviceRegister(gs) + } + defer func(gs *grpc.Server) { + if err != nil { + sctx.lg.Warn("stopping insecure grpc server due to error", zap.Error(err)) + gs.Stop() + sctx.lg.Warn("stopped insecure grpc server due to error", zap.Error(err)) + } + }(gs) + } + if onlyGRPC { + server = func() error { + return gs.Serve(sctx.l) + } + } else { + server = m.Serve + + httpl := m.Match(cmux.HTTP1()) + sctx.wg.Add(1) + go func(srvhttp *http.Server, tlsLis net.Listener) { + defer sctx.wg.Done() + errHandler(srvhttp.Serve(tlsLis)) + }(srv, httpl) + + if grpcEnabled { + grpcl := m.Match(cmux.HTTP2()) + sctx.wg.Add(1) + go func(gs *grpc.Server, l net.Listener) { + defer sctx.wg.Done() + errHandler(gs.Serve(l)) + }(gs, grpcl) + } } - httpl := m.Match(cmux.HTTP1()) - go func() { errHandler(srvhttp.Serve(httpl)) }() - sctx.serversC <- &servers{grpc: gs, http: srvhttp} + sctx.serversC <- &servers{grpc: gs, http: srv} sctx.lg.Info( "serving client traffic insecurely; this is strongly discouraged!", + zap.String("traffic", traffic), zap.String("address", sctx.l.Addr().String()), ) } if sctx.secure { + var gs *grpc.Server + var srv *http.Server + tlscfg, tlsErr := tlsinfo.ServerConfig() if tlsErr != nil { return tlsErr } - gs = v3rpc.Server(s, tlscfg, gopts...) - v3electionpb.RegisterElectionServer(gs, servElection) - v3lockpb.RegisterLockServer(gs, servLock) - if sctx.serviceRegister != nil { - sctx.serviceRegister(gs) + + if grpcEnabled { + gs = v3rpc.Server(s, tlscfg, nil, gopts...) + v3electionpb.RegisterElectionServer(gs, servElection) + v3lockpb.RegisterLockServer(gs, servLock) + if sctx.serviceRegister != nil { + sctx.serviceRegister(gs) + } + defer func(gs *grpc.Server) { + if err != nil { + sctx.lg.Warn("stopping secure grpc server due to error", zap.Error(err)) + gs.Stop() + sctx.lg.Warn("stopped secure grpc server due to error", zap.Error(err)) + } + }(gs) } - handler = grpcHandlerFunc(gs, handler) - - var gwmux *gw.ServeMux - if s.Cfg.EnableGRPCGateway { - dtls := tlscfg.Clone() - // trust local server - dtls.InsecureSkipVerify = true - bundle := credentials.NewBundle(credentials.Config{TLSConfig: dtls}) - opts := []grpc.DialOption{grpc.WithTransportCredentials(bundle.TransportCredentials())} - gwmux, err = sctx.registerGateway(opts) - if err != nil { + if httpEnabled { + if grpcEnabled { + handler = grpcHandlerFunc(gs, handler) + } + httpmux := sctx.createMux(gwmux, handler) + + srv = &http.Server{ + Handler: createAccessController(sctx.lg, s, httpmux), + TLSConfig: tlscfg, + ErrorLog: logger, // do not log user error + } + if err := configureHttpServer(srv, s.Cfg); err != nil { + sctx.lg.Error("Configure https server failed", zap.Error(err)) return err } } - var tlsl net.Listener - tlsl, err = transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo) - if err != nil { - return err - } - // TODO: add debug flag; enable logging when debug flag is set - httpmux := sctx.createMux(gwmux, handler) + if onlyGRPC { + server = func() error { return gs.Serve(sctx.l) } + } else { + server = m.Serve - srv := &http.Server{ - Handler: createAccessController(sctx.lg, s, httpmux), - TLSConfig: tlscfg, - ErrorLog: logger, // do not log user error + tlsl, tlsErr := transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo) + if tlsErr != nil { + return tlsErr + } + sctx.wg.Add(1) + go func(srvhttp *http.Server, tlsl net.Listener) { + defer sctx.wg.Done() + errHandler(srvhttp.Serve(tlsl)) + }(srv, tlsl) } - go func() { errHandler(srv.Serve(tlsl)) }() sctx.serversC <- &servers{secure: true, grpc: gs, http: srv} sctx.lg.Info( "serving client traffic securely", + zap.String("traffic", traffic), zap.String("address", sctx.l.Addr().String()), ) } - close(sctx.serversC) - return m.Serve() + err = server() + sctx.close() + // ensure all goroutines, which are created by this method, to complete before this method returns. + sctx.wg.Wait() + return err +} + +func configureHttpServer(srv *http.Server, cfg config.ServerConfig) error { + // todo (ahrtr): should we support configuring other parameters in the future as well? + return http2.ConfigureServer(srv, &http2.Server{ + MaxConcurrentStreams: cfg.MaxConcurrentStreams, + }) } // grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC @@ -213,16 +312,10 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha type registerHandlerFunc func(context.Context, *gw.ServeMux, *grpc.ClientConn) error -func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, error) { +func (sctx *serveCtx) registerGateway(dial func(ctx context.Context) (*grpc.ClientConn, error)) (*gw.ServeMux, error) { ctx := sctx.ctx - addr := sctx.addr - if network := sctx.network; network == "unix" { - // explicitly define unix network for gRPC socket support - addr = fmt.Sprintf("%s://%s", network, addr) - } - - conn, err := grpc.DialContext(ctx, addr, opts...) + conn, err := dial(ctx) if err != nil { return nil, err } @@ -243,7 +336,9 @@ func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, err return nil, err } } + sctx.wg.Add(1) go func() { + defer sctx.wg.Done() <-ctx.Done() if cerr := conn.Close(); cerr != nil { sctx.lg.Warn( @@ -257,6 +352,18 @@ func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, err return gwmux, nil } +type wsProxyZapLogger struct { + *zap.Logger +} + +func (w wsProxyZapLogger) Warnln(i ...interface{}) { + w.Warn(fmt.Sprint(i...)) +} + +func (w wsProxyZapLogger) Debugln(i ...interface{}) { + w.Debug(fmt.Sprint(i...)) +} + func (sctx *serveCtx) createMux(gwmux *gw.ServeMux, handler http.Handler) *http.ServeMux { httpmux := http.NewServeMux() for path, h := range sctx.userHandlers { @@ -276,6 +383,7 @@ func (sctx *serveCtx) createMux(gwmux *gw.ServeMux, handler http.Handler) *http. }, ), wsproxy.WithMaxRespBodyBufferSize(0x7fffffff), + wsproxy.WithLogger(wsProxyZapLogger{sctx.lg}), ), ) } @@ -420,3 +528,9 @@ func (sctx *serveCtx) registerTrace() { evf := func(w http.ResponseWriter, r *http.Request) { trace.RenderEvents(w, r, true) } sctx.registerUserHandler("/debug/events", http.HandlerFunc(evf)) } + +func (sctx *serveCtx) close() { + sctx.closeOnce.Do(func() { + close(sctx.serversC) + }) +} diff --git a/server/embed/serve_test.go b/server/embed/serve_test.go index aada585f07e9..6a1991119eed 100644 --- a/server/embed/serve_test.go +++ b/server/embed/serve_test.go @@ -38,8 +38,8 @@ func TestStartEtcdWrongToken(t *testing.T) { urls := newEmbedURLs(2) curls := []url.URL{urls[0]} purls := []url.URL{urls[1]} - cfg.LCUrls, cfg.ACUrls = curls, curls - cfg.LPUrls, cfg.APUrls = purls, purls + cfg.ListenClientUrls, cfg.AdvertiseClientUrls = curls, curls + cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = purls, purls cfg.InitialCluster = "" for i := range purls { cfg.InitialCluster += ",default=" + purls[i].String() diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index f9c91d9f9c52..349590ccfdc9 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -26,15 +26,19 @@ import ( "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/logutil" + "go.etcd.io/etcd/client/pkg/v3/tlsutil" "go.etcd.io/etcd/pkg/v3/flags" cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" + "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.uber.org/zap" "sigs.k8s.io/yaml" ) +const deprecatedWarningMessage = "--%s is deprecated in 3.5 and will be decommissioned in 3.6." + var ( proxyFlagOff = "off" proxyFlagReadonly = "readonly" @@ -61,6 +65,17 @@ var ( "test.coverprofile", "test.outputdir", } + + deprecatedFlags = map[string]struct{}{ + "enable-v2": struct{}{}, + "experimental-enable-v2v3": struct{}{}, + "proxy": struct{}{}, + "proxy-failure-wait": struct{}{}, + "proxy-refresh-interval": struct{}{}, + "proxy-dial-timeout": struct{}{}, + "proxy-write-timeout": struct{}{}, + "proxy-read-timeout": struct{}{}, + } ) type configProxy struct { @@ -145,7 +160,11 @@ func newConfig() *config { ) fs.Var( flags.NewUniqueURLsWithExceptions(embed.DefaultListenClientURLs, ""), "listen-client-urls", - "List of URLs to listen on for client traffic.", + "List of URLs to listen on for client grpc traffic and http as long as --listen-client-http-urls is not specified.", + ) + fs.Var( + flags.NewUniqueURLsWithExceptions("", ""), "listen-client-http-urls", + "List of URLs to listen on for http only client traffic. Enabling this flag removes http services from --listen-client-urls.", ) fs.Var( flags.NewUniqueURLsWithExceptions("", ""), @@ -171,6 +190,8 @@ func newConfig() *config { fs.BoolVar(&cfg.ec.SocketOpts.ReusePort, "socket-reuse-port", cfg.ec.SocketOpts.ReusePort, "Enable to set socket option SO_REUSEPORT on listeners allowing rebinding of a port already in use.") fs.BoolVar(&cfg.ec.SocketOpts.ReuseAddress, "socket-reuse-address", cfg.ec.SocketOpts.ReuseAddress, "Enable to set socket option SO_REUSEADDR on listeners allowing binding to an address in `TIME_WAIT` state.") + fs.Var(flags.NewUint32Value(cfg.ec.MaxConcurrentStreams), "max-concurrent-streams", "Maximum concurrent streams that each client can open at a time.") + // raft connection timeouts fs.DurationVar(&rafthttp.ConnReadTimeout, "raft-read-timeout", rafthttp.DefaultConnReadTimeout, "Read timeout set on each rafthttp connection") fs.DurationVar(&rafthttp.ConnWriteTimeout, "raft-write-timeout", rafthttp.DefaultConnWriteTimeout, "Write timeout set on each rafthttp connection") @@ -194,23 +215,23 @@ func newConfig() *config { fs.StringVar(&cfg.ec.DNSClusterServiceName, "discovery-srv-name", cfg.ec.DNSClusterServiceName, "Service name to query when using DNS discovery.") fs.StringVar(&cfg.ec.InitialCluster, "initial-cluster", cfg.ec.InitialCluster, "Initial cluster configuration for bootstrapping.") fs.StringVar(&cfg.ec.InitialClusterToken, "initial-cluster-token", cfg.ec.InitialClusterToken, "Initial cluster token for the etcd cluster during bootstrap.") - fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' or 'existing').") + fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' when bootstrapping a new cluster or 'existing' when adding new members to an existing cluster). After successful initialization (bootstrapping or adding), flag is ignored on restarts.") fs.BoolVar(&cfg.ec.StrictReconfigCheck, "strict-reconfig-check", cfg.ec.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.") fs.BoolVar(&cfg.ec.PreVote, "pre-vote", cfg.ec.PreVote, "Enable to run an additional Raft election phase.") - fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.") - fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.") + fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5 and will be decommissioned in v3.6.") + fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5 and will be decommissioned in 3.6.") fs.Var(cfg.cf.v2deprecation, "v2-deprecation", fmt.Sprintf("v2store deprecation stage: %q. ", cfg.cf.proxy.Valids())) // proxy fs.Var(cfg.cf.proxy, "proxy", fmt.Sprintf("Valid values include %q", cfg.cf.proxy.Valids())) - fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state.") - fs.UintVar(&cfg.cp.ProxyRefreshIntervalMs, "proxy-refresh-interval", cfg.cp.ProxyRefreshIntervalMs, "Time (in milliseconds) of the endpoints refresh interval.") - fs.UintVar(&cfg.cp.ProxyDialTimeoutMs, "proxy-dial-timeout", cfg.cp.ProxyDialTimeoutMs, "Time (in milliseconds) for a dial to timeout.") - fs.UintVar(&cfg.cp.ProxyWriteTimeoutMs, "proxy-write-timeout", cfg.cp.ProxyWriteTimeoutMs, "Time (in milliseconds) for a write to timeout.") - fs.UintVar(&cfg.cp.ProxyReadTimeoutMs, "proxy-read-timeout", cfg.cp.ProxyReadTimeoutMs, "Time (in milliseconds) for a read to timeout.") + fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyRefreshIntervalMs, "proxy-refresh-interval", cfg.cp.ProxyRefreshIntervalMs, "Time (in milliseconds) of the endpoints refresh interval. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyDialTimeoutMs, "proxy-dial-timeout", cfg.cp.ProxyDialTimeoutMs, "Time (in milliseconds) for a dial to timeout. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyWriteTimeoutMs, "proxy-write-timeout", cfg.cp.ProxyWriteTimeoutMs, "Time (in milliseconds) for a write to timeout. Deprecated in 3.5 and will be decommissioned in 3.6.") + fs.UintVar(&cfg.cp.ProxyReadTimeoutMs, "proxy-read-timeout", cfg.cp.ProxyReadTimeoutMs, "Time (in milliseconds) for a read to timeout. Deprecated in 3.5 and will be decommissioned in 3.6.") // security fs.StringVar(&cfg.ec.ClientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.") @@ -219,7 +240,7 @@ func newConfig() *config { fs.StringVar(&cfg.ec.ClientTLSInfo.ClientKeyFile, "client-key-file", "", "Path to an explicit peer client TLS key file otherwise key file will be used when client auth is required.") fs.BoolVar(&cfg.ec.ClientTLSInfo.ClientCertAuth, "client-cert-auth", false, "Enable client cert authentication.") fs.StringVar(&cfg.ec.ClientTLSInfo.CRLFile, "client-crl-file", "", "Path to the client certificate revocation list file.") - fs.StringVar(&cfg.ec.ClientTLSInfo.AllowedHostname, "client-cert-allowed-hostname", "", "Allowed TLS hostname for client cert authentication.") + fs.Var(flags.NewStringsValue(""), "client-cert-allowed-hostname", "Comma-separated list of allowed SAN hostnames for client cert authentication.") fs.StringVar(&cfg.ec.ClientTLSInfo.TrustedCAFile, "trusted-ca-file", "", "Path to the client server TLS trusted CA cert file.") fs.BoolVar(&cfg.ec.ClientAutoTLS, "auto-tls", false, "Client TLS using generated certificates") fs.StringVar(&cfg.ec.PeerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.") @@ -231,10 +252,12 @@ func newConfig() *config { fs.BoolVar(&cfg.ec.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates") fs.UintVar(&cfg.ec.SelfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the client and peer certificates, unit is year") fs.StringVar(&cfg.ec.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.") - fs.StringVar(&cfg.ec.PeerTLSInfo.AllowedCN, "peer-cert-allowed-cn", "", "Allowed CN for inter peer authentication.") - fs.StringVar(&cfg.ec.PeerTLSInfo.AllowedHostname, "peer-cert-allowed-hostname", "", "Allowed TLS hostname for inter peer authentication.") + fs.Var(flags.NewStringsValue(""), "peer-cert-allowed-cn", "Comma-separated list of allowed CNs for inter-peer TLS authentication.") + fs.Var(flags.NewStringsValue(""), "peer-cert-allowed-hostname", "Comma-separated list of allowed SAN hostnames for inter-peer TLS authentication.") fs.Var(flags.NewStringsValue(""), "cipher-suites", "Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).") fs.BoolVar(&cfg.ec.PeerTLSInfo.SkipClientSANVerify, "experimental-peer-skip-client-san-verification", false, "Skip verification of SAN field in client certificate for peer connections.") + fs.StringVar(&cfg.ec.TlsMinVersion, "tls-min-version", string(tlsutil.TLSVersion12), "Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.") + fs.StringVar(&cfg.ec.TlsMaxVersion, "tls-max-version", string(tlsutil.TLSVersionDefault), "Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty defers to Go).") fs.Var( flags.NewUniqueURLsWithExceptions("*", "*"), @@ -267,6 +290,7 @@ func newConfig() *config { fs.StringVar(&cfg.ec.ExperimentalDistributedTracingAddress, "experimental-distributed-tracing-address", embed.ExperimentalDistributedTracingAddress, "Address for distributed tracing used for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag).") fs.StringVar(&cfg.ec.ExperimentalDistributedTracingServiceName, "experimental-distributed-tracing-service-name", embed.ExperimentalDistributedTracingServiceName, "Configures service name for distributed tracing to be used to define service name for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). 'etcd' is the default service name. Use the same service name for all instances of etcd.") fs.StringVar(&cfg.ec.ExperimentalDistributedTracingServiceInstanceID, "experimental-distributed-tracing-instance-id", "", "Configures service instance ID for distributed tracing to be used to define service instance ID key for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag). There is no default value set. This ID must be unique per etcd instance.") + fs.IntVar(&cfg.ec.ExperimentalDistributedTracingSamplingRatePerMillion, "experimental-distributed-tracing-sampling-rate", 0, "Number of samples to collect per million spans for OpenTelemetry Tracing (if enabled with experimental-enable-distributed-tracing flag).") // auth fs.StringVar(&cfg.ec.AuthToken, "auth-token", cfg.ec.AuthToken, "Specify auth token specific options.") @@ -279,19 +303,28 @@ func newConfig() *config { // experimental fs.BoolVar(&cfg.ec.ExperimentalInitialCorruptCheck, "experimental-initial-corrupt-check", cfg.ec.ExperimentalInitialCorruptCheck, "Enable to check data corruption before serving any client/peer traffic.") fs.DurationVar(&cfg.ec.ExperimentalCorruptCheckTime, "experimental-corrupt-check-time", cfg.ec.ExperimentalCorruptCheckTime, "Duration of time between cluster corruption check passes.") + fs.BoolVar(&cfg.ec.ExperimentalCompactHashCheckEnabled, "experimental-compact-hash-check-enabled", cfg.ec.ExperimentalCompactHashCheckEnabled, "Enable leader to periodically check followers compaction hashes.") + fs.DurationVar(&cfg.ec.ExperimentalCompactHashCheckTime, "experimental-compact-hash-check-time", cfg.ec.ExperimentalCompactHashCheckTime, "Duration of time between leader checks followers compaction hashes.") - fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable to persist lease remaining TTL to prevent indefinite auto-renewal of long lived leases.") + fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change.") + // TODO: delete in v3.7 + fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpointPersist, "experimental-enable-lease-checkpoint-persist", false, "Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled.") fs.IntVar(&cfg.ec.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ec.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.") + fs.DurationVar(&cfg.ec.ExperimentalCompactionSleepInterval, "experimental-compaction-sleep-interval", cfg.ec.ExperimentalCompactionSleepInterval, "Sets the sleep interval between each compaction batch.") fs.DurationVar(&cfg.ec.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ec.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.") fs.DurationVar(&cfg.ec.ExperimentalDowngradeCheckTime, "experimental-downgrade-check-time", cfg.ec.ExperimentalDowngradeCheckTime, "Duration of time between two downgrade status check.") fs.DurationVar(&cfg.ec.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ec.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.") fs.BoolVar(&cfg.ec.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ec.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.BoolVar(&cfg.ec.ExperimentalTxnModeWriteWithSharedBuffer, "experimental-txn-mode-write-with-shared-buffer", true, "Enable the write transaction to use a shared buffer in its readonly check operations.") + fs.BoolVar(&cfg.ec.ExperimentalStopGRPCServiceOnDefrag, "experimental-stop-grpc-service-on-defrag", cfg.ec.ExperimentalStopGRPCServiceOnDefrag, "Enable etcd gRPC service to stop serving client requests on defragmentation.") fs.UintVar(&cfg.ec.ExperimentalBootstrapDefragThresholdMegabytes, "experimental-bootstrap-defrag-threshold-megabytes", 0, "Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect.") + fs.IntVar(&cfg.ec.ExperimentalMaxLearners, "experimental-max-learners", membership.DefaultMaxLearners, "Sets the maximum number of learners that can be available in the cluster membership.") + fs.Uint64Var(&cfg.ec.SnapshotCatchUpEntries, "experimental-snapshot-catchup-entries", cfg.ec.SnapshotCatchUpEntries, "(WARNING: Use this flag with caution!) Number of entries for a slow follower to catch up after compacting the raft storage entries.") // unsafe fs.BoolVar(&cfg.ec.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.") fs.BoolVar(&cfg.ec.ForceNewCluster, "force-new-cluster", false, "Force to create a new one member cluster.") + fs.Uint64Var(&cfg.ec.ForceNewClusterBumpAmount, "force-new-cluster-bump-amount", 0, "How much to increase the latest revision after --force-new-cluster.") // ignored for _, f := range cfg.ignored { @@ -347,17 +380,30 @@ func (cfg *config) parse(arguments []string) error { cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT } + var warningsForDeprecatedFlags []string + cfg.cf.flagSet.Visit(func(f *flag.Flag) { + if _, ok := deprecatedFlags[f.Name]; ok { + warningsForDeprecatedFlags = append(warningsForDeprecatedFlags, fmt.Sprintf(deprecatedWarningMessage, f.Name)) + } + }) + if len(warningsForDeprecatedFlags) > 0 { + if lg := cfg.ec.GetLogger(); lg != nil { + for _, msg := range warningsForDeprecatedFlags { + lg.Warn(msg) + } + } + } + // now logger is set up return err } func (cfg *config) configFromCmdLine() error { // user-specified logger is not setup yet, use this logger during flag parsing - lg, err := zap.NewProduction() + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { return err } - verKey := "ETCD_VERSION" if verVal := os.Getenv(verKey); verVal != "" { // unset to avoid any possible side-effect. @@ -384,17 +430,24 @@ func (cfg *config) configFromCmdLine() error { lg.Info(fmt.Sprintf("raft-write-timeout increased to minimum value: %v", rafthttp.DefaultConnWriteTimeout)) } - cfg.ec.LPUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-peer-urls") - cfg.ec.APUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "initial-advertise-peer-urls") - cfg.ec.LCUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-urls") - cfg.ec.ACUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "advertise-client-urls") + cfg.ec.ListenPeerUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-peer-urls") + cfg.ec.AdvertisePeerUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "initial-advertise-peer-urls") + cfg.ec.ListenClientUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-urls") + cfg.ec.ListenClientHttpUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-http-urls") + cfg.ec.AdvertiseClientUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "advertise-client-urls") cfg.ec.ListenMetricsUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-metrics-urls") cfg.ec.CORS = flags.UniqueURLsMapFromFlag(cfg.cf.flagSet, "cors") cfg.ec.HostWhitelist = flags.UniqueStringsMapFromFlag(cfg.cf.flagSet, "host-whitelist") + cfg.ec.ClientTLSInfo.AllowedHostnames = flags.StringsFromFlag(cfg.cf.flagSet, "client-cert-allowed-hostname") + cfg.ec.PeerTLSInfo.AllowedCNs = flags.StringsFromFlag(cfg.cf.flagSet, "peer-cert-allowed-cn") + cfg.ec.PeerTLSInfo.AllowedHostnames = flags.StringsFromFlag(cfg.cf.flagSet, "peer-cert-allowed-hostname") + cfg.ec.CipherSuites = flags.StringsFromFlag(cfg.cf.flagSet, "cipher-suites") + cfg.ec.MaxConcurrentStreams = flags.Uint32FromFlag(cfg.cf.flagSet, "max-concurrent-streams") + cfg.ec.LogOutputs = flags.UniqueStringsFromFlag(cfg.cf.flagSet, "log-outputs") cfg.ec.ClusterState = cfg.cf.clusterState.String() @@ -406,7 +459,7 @@ func (cfg *config) configFromCmdLine() error { // disable default advertise-client-urls if lcurls is set missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls") if !cfg.mayBeProxy() && missingAC { - cfg.ec.ACUrls = nil + cfg.ec.AdvertiseClientUrls = nil } // disable default initial-cluster if discovery is set diff --git a/server/etcdmain/config_test.go b/server/etcdmain/config_test.go index 0dd4db97ec26..7ace5066a996 100644 --- a/server/etcdmain/config_test.go +++ b/server/etcdmain/config_test.go @@ -15,6 +15,7 @@ package etcdmain import ( + "flag" "fmt" "io/ioutil" "net/url" @@ -23,6 +24,7 @@ import ( "strings" "testing" + "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/server/v3/embed" "sigs.k8s.io/yaml" ) @@ -36,6 +38,7 @@ func TestConfigParsingMemberFlags(t *testing.T) { "-snapshot-count=10", "-listen-peer-urls=http://localhost:8000,https://localhost:8001", "-listen-client-urls=http://localhost:7000,https://localhost:7001", + "-listen-client-http-urls=http://localhost:7002,https://localhost:7003", // it should be set if -listen-client-urls is set "-advertise-client-urls=http://localhost:7000,https://localhost:7001", } @@ -51,14 +54,15 @@ func TestConfigParsingMemberFlags(t *testing.T) { func TestConfigFileMemberFields(t *testing.T) { yc := struct { - Dir string `json:"data-dir"` - MaxSnapFiles uint `json:"max-snapshots"` - MaxWalFiles uint `json:"max-wals"` - Name string `json:"name"` - SnapshotCount uint64 `json:"snapshot-count"` - LPUrls string `json:"listen-peer-urls"` - LCUrls string `json:"listen-client-urls"` - AcurlsCfgFile string `json:"advertise-client-urls"` + Dir string `json:"data-dir"` + MaxSnapFiles uint `json:"max-snapshots"` + MaxWalFiles uint `json:"max-wals"` + Name string `json:"name"` + SnapshotCount uint64 `json:"snapshot-count"` + ListenPeerUrls string `json:"listen-peer-urls"` + ListenClientUrls string `json:"listen-client-urls"` + ListenClientHttpUrls string `json:"listen-client-http-urls"` + AdvertiseClientUrls string `json:"advertise-client-urls"` }{ "testdir", 10, @@ -67,6 +71,7 @@ func TestConfigFileMemberFields(t *testing.T) { 10, "http://localhost:8000,https://localhost:8001", "http://localhost:7000,https://localhost:7001", + "http://localhost:7002,https://localhost:7003", "http://localhost:7000,https://localhost:7001", } @@ -493,6 +498,24 @@ func TestConfigFileElectionTimeout(t *testing.T) { } } +func TestFlagsPresentInHelp(t *testing.T) { + cfg := newConfig() + cfg.cf.flagSet.VisitAll(func(f *flag.Flag) { + if _, ok := f.Value.(*flags.IgnoredFlag); ok { + // Ignored flags do not need to be in the help + return + } + if f.Name == "experimental-snapshot-catchup-entries" { + // Ignore because it is supposed to only be used in tests. + return + } + flagText := fmt.Sprintf("--%s", f.Name) + if !strings.Contains(flagsline, flagText) && !strings.Contains(usageline, flagText) { + t.Errorf("Neither flagsline nor usageline in help.go contains flag named %s", flagText) + } + }) +} + func mustCreateCfgFile(t *testing.T, b []byte) *os.File { tmpfile, err := ioutil.TempFile("", "servercfg") if err != nil { @@ -513,13 +536,14 @@ func mustCreateCfgFile(t *testing.T, b []byte) *os.File { func validateMemberFlags(t *testing.T, cfg *config) { wcfg := &embed.Config{ - Dir: "testdir", - LPUrls: []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}}, - LCUrls: []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}}, - MaxSnapFiles: 10, - MaxWalFiles: 10, - Name: "testname", - SnapshotCount: 10, + Dir: "testdir", + ListenPeerUrls: []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}}, + ListenClientUrls: []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}}, + ListenClientHttpUrls: []url.URL{{Scheme: "http", Host: "localhost:7002"}, {Scheme: "https", Host: "localhost:7003"}}, + MaxSnapFiles: 10, + MaxWalFiles: 10, + Name: "testname", + SnapshotCount: 10, } if cfg.ec.Dir != wcfg.Dir { @@ -537,18 +561,21 @@ func validateMemberFlags(t *testing.T, cfg *config) { if cfg.ec.SnapshotCount != wcfg.SnapshotCount { t.Errorf("snapcount = %v, want %v", cfg.ec.SnapshotCount, wcfg.SnapshotCount) } - if !reflect.DeepEqual(cfg.ec.LPUrls, wcfg.LPUrls) { - t.Errorf("listen-peer-urls = %v, want %v", cfg.ec.LPUrls, wcfg.LPUrls) + if !reflect.DeepEqual(cfg.ec.ListenPeerUrls, wcfg.ListenPeerUrls) { + t.Errorf("listen-peer-urls = %v, want %v", cfg.ec.ListenPeerUrls, wcfg.ListenPeerUrls) + } + if !reflect.DeepEqual(cfg.ec.ListenClientUrls, wcfg.ListenClientUrls) { + t.Errorf("listen-client-urls = %v, want %v", cfg.ec.ListenClientUrls, wcfg.ListenClientUrls) } - if !reflect.DeepEqual(cfg.ec.LCUrls, wcfg.LCUrls) { - t.Errorf("listen-client-urls = %v, want %v", cfg.ec.LCUrls, wcfg.LCUrls) + if !reflect.DeepEqual(cfg.ec.ListenClientHttpUrls, wcfg.ListenClientHttpUrls) { + t.Errorf("listen-client-http-urls = %v, want %v", cfg.ec.ListenClientHttpUrls, wcfg.ListenClientHttpUrls) } } func validateClusteringFlags(t *testing.T, cfg *config) { wcfg := newConfig() - wcfg.ec.APUrls = []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}} - wcfg.ec.ACUrls = []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}} + wcfg.ec.AdvertisePeerUrls = []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}} + wcfg.ec.AdvertiseClientUrls = []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}} wcfg.ec.ClusterState = embed.ClusterStateFlagExisting wcfg.cf.fallback.Set(fallbackFlagExit) wcfg.ec.InitialCluster = "0=http://localhost:8000" @@ -566,11 +593,11 @@ func validateClusteringFlags(t *testing.T, cfg *config) { if cfg.ec.InitialClusterToken != wcfg.ec.InitialClusterToken { t.Errorf("initialClusterToken = %v, want %v", cfg.ec.InitialClusterToken, wcfg.ec.InitialClusterToken) } - if !reflect.DeepEqual(cfg.ec.APUrls, wcfg.ec.APUrls) { - t.Errorf("initial-advertise-peer-urls = %v, want %v", cfg.ec.APUrls, wcfg.ec.APUrls) + if !reflect.DeepEqual(cfg.ec.AdvertisePeerUrls, wcfg.ec.AdvertisePeerUrls) { + t.Errorf("initial-advertise-peer-urls = %v, want %v", cfg.ec.AdvertisePeerUrls, wcfg.ec.AdvertisePeerUrls) } - if !reflect.DeepEqual(cfg.ec.ACUrls, wcfg.ec.ACUrls) { - t.Errorf("advertise-client-urls = %v, want %v", cfg.ec.ACUrls, wcfg.ec.ACUrls) + if !reflect.DeepEqual(cfg.ec.AdvertiseClientUrls, wcfg.ec.AdvertiseClientUrls) { + t.Errorf("advertise-client-urls = %v, want %v", cfg.ec.AdvertiseClientUrls, wcfg.ec.AdvertiseClientUrls) } } diff --git a/server/etcdmain/etcd.go b/server/etcdmain/etcd.go index eb2585699dc0..9921b89df366 100644 --- a/server/etcdmain/etcd.go +++ b/server/etcdmain/etcd.go @@ -27,6 +27,7 @@ import ( "time" "go.etcd.io/etcd/client/pkg/v3/fileutil" + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" pkgioutil "go.etcd.io/etcd/pkg/v3/ioutil" @@ -63,7 +64,7 @@ func startEtcdOrProxyV2(args []string) { if lg == nil { var zapError error // use this logger - lg, zapError = zap.NewProduction() + lg, zapError = logutil.CreateDefaultZapLogger(zap.InfoLevel) if zapError != nil { fmt.Printf("error creating zap logger %v", zapError) os.Exit(1) @@ -192,7 +193,7 @@ func startEtcdOrProxyV2(args []string) { if cfg.ec.InitialCluster == cfg.ec.InitialClusterFromName(cfg.ec.Name) { lg.Warn("forgot to set --initial-cluster?") } - if types.URLs(cfg.ec.APUrls).String() == embed.DefaultInitialAdvertisePeerURLs { + if types.URLs(cfg.ec.AdvertisePeerUrls).String() == embed.DefaultInitialAdvertisePeerURLs { lg.Warn("forgot to set --initial-advertise-peer-urls?") } if cfg.ec.InitialCluster == cfg.ec.InitialClusterFromName(cfg.ec.Name) && len(cfg.ec.Durl) == 0 { @@ -275,7 +276,7 @@ func startProxy(cfg *config) error { } cfg.ec.Dir = filepath.Join(cfg.ec.Dir, "proxy") - err = fileutil.TouchDirAll(cfg.ec.Dir) + err = fileutil.TouchDirAll(lg, cfg.ec.Dir) if err != nil { return err } @@ -388,11 +389,11 @@ func startProxy(cfg *config) error { // setup self signed certs when serving https cHosts, cTLS := []string{}, false - for _, u := range cfg.ec.LCUrls { + for _, u := range cfg.ec.ListenClientUrls { cHosts = append(cHosts, u.Host) cTLS = cTLS || u.Scheme == "https" } - for _, u := range cfg.ec.ACUrls { + for _, u := range cfg.ec.AdvertiseClientUrls { cHosts = append(cHosts, u.Host) cTLS = cTLS || u.Scheme == "https" } @@ -405,7 +406,7 @@ func startProxy(cfg *config) error { } // Start a proxy server goroutine for each listen address - for _, u := range cfg.ec.LCUrls { + for _, u := range cfg.ec.ListenClientUrls { l, err := transport.NewListener(u.Host, u.Scheme, &listenerTLS) if err != nil { return err @@ -415,7 +416,7 @@ func startProxy(cfg *config) error { go func() { lg.Info("v2 proxy started listening on client requests", zap.String("host", host)) mux := http.NewServeMux() - etcdhttp.HandlePrometheus(mux) // v2 proxy just uses the same port + etcdhttp.HandleMetrics(mux) // v2 proxy just uses the same port mux.Handle("/", ph) lg.Fatal("done serving", zap.Error(http.Serve(l, mux))) }() @@ -463,6 +464,10 @@ func identifyDataDirOrDie(lg *zap.Logger, dir string) dirType { } func checkSupportArch() { + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) + if err != nil { + panic(err) + } // to add a new platform, check https://github.com/etcd-io/website/blob/main/content/en/docs/next/op-guide/supported-platform.md if runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64" || @@ -474,10 +479,10 @@ func checkSupportArch() { // so unset here to not parse through flag defer os.Unsetenv("ETCD_UNSUPPORTED_ARCH") if env, ok := os.LookupEnv("ETCD_UNSUPPORTED_ARCH"); ok && env == runtime.GOARCH { - fmt.Printf("running etcd on unsupported architecture %q since ETCD_UNSUPPORTED_ARCH is set\n", env) + lg.Info("running etcd on unsupported architecture since ETCD_UNSUPPORTED_ARCH is set", zap.String("arch", env)) return } - fmt.Printf("etcd on unsupported platform without ETCD_UNSUPPORTED_ARCH=%s set\n", runtime.GOARCH) + lg.Error("running etcd on unsupported architecture since ETCD_UNSUPPORTED_ARCH is set", zap.String("arch", runtime.GOARCH)) os.Exit(1) } diff --git a/server/etcdmain/gateway.go b/server/etcdmain/gateway.go index 3e4d0620c667..64fb90df2c12 100644 --- a/server/etcdmain/gateway.go +++ b/server/etcdmain/gateway.go @@ -21,6 +21,7 @@ import ( "os" "time" + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/server/v3/proxy/tcpproxy" "github.com/spf13/cobra" @@ -92,8 +93,7 @@ func stripSchema(eps []string) []string { } func startGateway(cmd *cobra.Command, args []string) { - var lg *zap.Logger - lg, err := zap.NewProduction() + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index e251e1825a80..a684d0b5e55a 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -31,6 +31,7 @@ import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/logutil" + "go.etcd.io/etcd/client/pkg/v3/tlsutil" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/leasing" @@ -46,21 +47,27 @@ import ( "github.com/soheilhy/cmux" "github.com/spf13/cobra" "go.uber.org/zap" + "go.uber.org/zap/zapgrpc" + "golang.org/x/net/http2" "google.golang.org/grpc" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/keepalive" ) var ( - grpcProxyListenAddr string - grpcProxyMetricsListenAddr string - grpcProxyEndpoints []string - grpcProxyDNSCluster string - grpcProxyDNSClusterServiceName string - grpcProxyInsecureDiscovery bool - grpcProxyDataDir string - grpcMaxCallSendMsgSize int - grpcMaxCallRecvMsgSize int + grpcProxyListenAddr string + grpcProxyMetricsListenAddr string + grpcProxyEndpoints []string + grpcProxyEndpointsAutoSyncInterval time.Duration + grpcProxyDialKeepAliveTime time.Duration + grpcProxyDialKeepAliveTimeout time.Duration + grpcProxyPermitWithoutStream bool + grpcProxyDNSCluster string + grpcProxyDNSClusterServiceName string + grpcProxyInsecureDiscovery bool + grpcProxyDataDir string + grpcMaxCallSendMsgSize int + grpcMaxCallRecvMsgSize int // tls for connecting to etcd @@ -71,11 +78,15 @@ var ( // tls for clients connecting to proxy - grpcProxyListenCA string - grpcProxyListenCert string - grpcProxyListenKey string - grpcProxyListenAutoTLS bool - grpcProxyListenCRL string + grpcProxyListenCA string + grpcProxyListenCert string + grpcProxyListenKey string + grpcProxyListenCipherSuites []string + grpcProxyListenAutoTLS bool + grpcProxyListenCRL string + grpcProxyListenTLSMinVersion string + grpcProxyListenTLSMaxVersion string + selfSignedCertValidity uint grpcProxyAdvertiseClientURL string @@ -94,6 +105,8 @@ var ( grpcKeepAliveMinTime time.Duration grpcKeepAliveTimeout time.Duration grpcKeepAliveInterval time.Duration + + maxConcurrentStreams uint32 ) const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024 @@ -126,6 +139,10 @@ func newGRPCProxyStartCommand() *cobra.Command { cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for endpoint /metrics requests on an additional interface") cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records") cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints") + cmd.Flags().DurationVar(&grpcProxyEndpointsAutoSyncInterval, "endpoints-auto-sync-interval", 0, "etcd endpoints auto sync interval (disabled by default)") + cmd.Flags().DurationVar(&grpcProxyDialKeepAliveTime, "dial-keepalive-time", 0, "keepalive time for client(grpc-proxy) connections (default 0, disable).") + cmd.Flags().DurationVar(&grpcProxyDialKeepAliveTimeout, "dial-keepalive-timeout", embed.DefaultGRPCKeepAliveTimeout, "keepalive timeout for client(grpc-proxy) connections (default 20s).") + cmd.Flags().BoolVar(&grpcProxyPermitWithoutStream, "permit-without-stream", false, "Enable client(grpc-proxy) to send keepalive pings even with no active RPCs.") cmd.Flags().StringVar(&grpcProxyAdvertiseClientURL, "advertise-client-url", "127.0.0.1:23790", "advertise address to register (must be reachable by client)") cmd.Flags().StringVar(&grpcProxyResolverPrefix, "resolver-prefix", "", "prefix to use for registering proxy (must be shared with other grpc-proxy members)") cmd.Flags().IntVar(&grpcProxyResolverTTL, "resolver-ttl", 0, "specify TTL, in seconds, when registering proxy endpoints") @@ -148,9 +165,12 @@ func newGRPCProxyStartCommand() *cobra.Command { cmd.Flags().StringVar(&grpcProxyListenCert, "cert-file", "", "identify secure connections to the proxy using this TLS certificate file") cmd.Flags().StringVar(&grpcProxyListenKey, "key-file", "", "identify secure connections to the proxy using this TLS key file") cmd.Flags().StringVar(&grpcProxyListenCA, "trusted-ca-file", "", "verify certificates of TLS-enabled secure proxy using this CA bundle") + cmd.Flags().StringSliceVar(&grpcProxyListenCipherSuites, "listen-cipher-suites", grpcProxyListenCipherSuites, "Comma-separated list of supported TLS cipher suites between client/proxy (empty will be auto-populated by Go).") cmd.Flags().BoolVar(&grpcProxyListenAutoTLS, "auto-tls", false, "proxy TLS using generated certificates") cmd.Flags().StringVar(&grpcProxyListenCRL, "client-crl-file", "", "proxy client certificate revocation list file.") cmd.Flags().UintVar(&selfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the proxy certificates, unit is year") + cmd.Flags().StringVar(&grpcProxyListenTLSMinVersion, "tls-min-version", string(tlsutil.TLSVersion12), "Minimum TLS version supported by grpc proxy. Possible values: TLS1.2, TLS1.3.") + cmd.Flags().StringVar(&grpcProxyListenTLSMaxVersion, "tls-max-version", string(tlsutil.TLSVersionDefault), "Maximum TLS version supported by grpc proxy. Possible values: TLS1.2, TLS1.3 (empty defers to Go).") // experimental flags cmd.Flags().BoolVar(&grpcProxyEnableOrdering, "experimental-serializable-ordering", false, "Ensure serializable reads have monotonically increasing store revisions across endpoints.") @@ -158,49 +178,66 @@ func newGRPCProxyStartCommand() *cobra.Command { cmd.Flags().BoolVar(&grpcProxyDebug, "debug", false, "Enable debug-level logging for grpc-proxy.") + cmd.Flags().Uint32Var(&maxConcurrentStreams, "max-concurrent-streams", math.MaxUint32, "Maximum concurrent streams that each client can open at a time.") + return &cmd } func startGRPCProxy(cmd *cobra.Command, args []string) { checkArgs() - - lcfg := logutil.DefaultZapLoggerConfig + lvl := zap.InfoLevel if grpcProxyDebug { - lcfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + lvl = zap.DebugLevel grpc.EnableTracing = true } - - lg, err := lcfg.Build() + lg, err := logutil.CreateDefaultZapLogger(lvl) if err != nil { - log.Fatal(err) + panic(err) } defer lg.Sync() - var gl grpclog.LoggerV2 - gl, err = logutil.NewGRPCLoggerV2(lcfg) - if err != nil { - log.Fatal(err) - } - grpclog.SetLoggerV2(gl) + grpclog.SetLoggerV2(zapgrpc.NewLogger(lg)) // The proxy itself (ListenCert) can have not-empty CN. // The empty CN is required for grpcProxyCert. // Please see https://github.com/etcd-io/etcd/issues/11970#issuecomment-687875315 for more context. - tlsinfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey, false) - - if tlsinfo == nil && grpcProxyListenAutoTLS { + tlsInfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey, false) + if tlsInfo == nil && grpcProxyListenAutoTLS { host := []string{"https://" + grpcProxyListenAddr} dir := filepath.Join(grpcProxyDataDir, "fixtures", "proxy") autoTLS, err := transport.SelfCert(lg, dir, host, selfSignedCertValidity) if err != nil { log.Fatal(err) } - tlsinfo = &autoTLS + tlsInfo = &autoTLS } - if tlsinfo != nil { - lg.Info("gRPC proxy server TLS", zap.String("tls-info", fmt.Sprintf("%+v", tlsinfo))) + if tlsInfo != nil { + if len(grpcProxyListenCipherSuites) > 0 { + cs, err := tlsutil.GetCipherSuites(grpcProxyListenCipherSuites) + if err != nil { + log.Fatal(err) + } + tlsInfo.CipherSuites = cs + } + if grpcProxyListenTLSMinVersion != "" { + version, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMinVersion) + if err != nil { + log.Fatal(err) + } + tlsInfo.MinVersion = version + } + if grpcProxyListenTLSMaxVersion != "" { + version, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMaxVersion) + if err != nil { + log.Fatal(err) + } + tlsInfo.MaxVersion = version + } + + lg.Info("gRPC proxy server TLS", zap.String("tls-info", fmt.Sprintf("%+v", tlsInfo))) } - m := mustListenCMux(lg, tlsinfo) + + m := mustListenCMux(lg, tlsInfo) grpcl := m.Match(cmux.HTTP2()) defer func() { grpcl.Close() @@ -213,17 +250,24 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { // TODO: The mechanism should be refactored to use internal connection. var proxyClient *clientv3.Client if grpcProxyAdvertiseClientURL != "" { - proxyClient = mustNewProxyClient(lg, tlsinfo) + proxyClient = mustNewProxyClient(lg, tlsInfo) } httpClient := mustNewHTTPClient(lg) - srvhttp, httpl := mustHTTPListener(lg, m, tlsinfo, client, proxyClient) + srvhttp, httpl := mustHTTPListener(lg, m, tlsInfo, client, proxyClient) + + if err := http2.ConfigureServer(srvhttp, &http2.Server{ + MaxConcurrentStreams: maxConcurrentStreams, + }); err != nil { + lg.Fatal("Failed to configure the http server", zap.Error(err)) + } + errc := make(chan error, 3) go func() { errc <- newGRPCProxyServer(lg, client).Serve(grpcl) }() go func() { errc <- srvhttp.Serve(httpl) }() go func() { errc <- m.Serve() }() if len(grpcProxyMetricsListenAddr) > 0 { - mhttpl := mustMetricsListener(lg, tlsinfo) + mhttpl := mustMetricsListener(lg, tlsInfo) go func() { mux := http.NewServeMux() grpcproxy.HandleMetrics(mux, httpClient, client.Endpoints()) @@ -266,6 +310,29 @@ func checkArgs() { fmt.Fprintln(os.Stderr, fmt.Errorf("selfSignedCertValidity is invalid,it should be greater than 0")) os.Exit(1) } + + minVersion, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMinVersion) + if err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("tls-min-version is invalid: %w", err)) + os.Exit(1) + } + maxVersion, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMaxVersion) + if err != nil { + fmt.Fprintln(os.Stderr, fmt.Errorf("tls-max-version is invalid: %w", err)) + os.Exit(1) + } + + // maxVersion == 0 means that Go selects the highest available version. + if maxVersion != 0 && minVersion > maxVersion { + fmt.Fprintln(os.Stderr, fmt.Errorf("min version (%s) is greater than max version (%s)", grpcProxyListenTLSMinVersion, grpcProxyListenTLSMaxVersion)) + os.Exit(1) + } + + // Check if user attempted to configure ciphers for TLS1.3 only: Go does not support that currently. + if minVersion == tls.VersionTLS13 && len(grpcProxyListenCipherSuites) > 0 { + fmt.Fprintln(os.Stderr, fmt.Errorf("cipher suites cannot be configured when only TLS1.3 is enabled")) + os.Exit(1) + } } func mustNewClient(lg *zap.Logger) *clientv3.Client { @@ -327,8 +394,9 @@ func newProxyClientCfg(lg *zap.Logger, eps []string, tls *transport.TLSInfo) (*c func newClientCfg(lg *zap.Logger, eps []string) (*clientv3.Config, error) { // set tls if any one tls option set cfg := clientv3.Config{ - Endpoints: eps, - DialTimeout: 5 * time.Second, + Endpoints: eps, + AutoSyncInterval: grpcProxyEndpointsAutoSyncInterval, + DialTimeout: 5 * time.Second, } if grpcMaxCallSendMsgSize > 0 { @@ -337,6 +405,13 @@ func newClientCfg(lg *zap.Logger, eps []string) (*clientv3.Config, error) { if grpcMaxCallRecvMsgSize > 0 { cfg.MaxCallRecvMsgSize = grpcMaxCallRecvMsgSize } + if grpcProxyDialKeepAliveTime > 0 { + cfg.DialKeepAliveTime = grpcProxyDialKeepAliveTime + } + if grpcProxyDialKeepAliveTimeout > 0 { + cfg.DialKeepAliveTimeout = grpcProxyDialKeepAliveTimeout + } + cfg.PermitWithoutStream = grpcProxyPermitWithoutStream tls := newTLS(grpcProxyCA, grpcProxyCert, grpcProxyKey, true) if tls == nil && grpcProxyInsecureSkipTLSVerify { @@ -361,7 +436,7 @@ func newTLS(ca, cert, key string, requireEmptyCN bool) *transport.TLSInfo { if ca == "" && cert == "" && key == "" { return nil } - return &transport.TLSInfo{TrustedCAFile: ca, CertFile: cert, KeyFile: key, EmptyCN: requireEmptyCN} + return &transport.TLSInfo{TrustedCAFile: ca, CertFile: cert, KeyFile: key} } func mustListenCMux(lg *zap.Logger, tlsinfo *transport.TLSInfo) cmux.CMux { diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index dc5b55fae7e7..cd6e2195873d 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -20,6 +20,7 @@ import ( cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" + "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "golang.org/x/crypto/bcrypt" ) @@ -63,7 +64,9 @@ Member: --listen-peer-urls 'http://localhost:2380' List of URLs to listen on for peer traffic. --listen-client-urls 'http://localhost:2379' - List of URLs to listen on for client traffic. + List of URLs to listen on for client grpc traffic and http as long as --listen-client-http-urls is not specified. + --listen-client-http-urls '' + List of URLs to listen on for http only client traffic. Enabling this flag removes http services from --listen-client-urls. --max-snapshots '` + strconv.Itoa(embed.DefaultMaxSnapshots) + `' Maximum number of snapshot files to retain (0 is unlimited). --max-wals '` + strconv.Itoa(embed.DefaultMaxWALs) + `' @@ -80,6 +83,10 @@ Member: Maximum number of operations permitted in a transaction. --max-request-bytes '1572864' Maximum client request size in bytes the server will accept. + --max-concurrent-streams 'math.MaxUint32' + Maximum concurrent streams that each client can open at a time. + --enable-grpc-gateway + Enable GRPC gateway. --grpc-keepalive-min-time '5s' Minimum duration interval that a client should wait before pinging server. --grpc-keepalive-interval '2h' @@ -89,7 +96,11 @@ Member: --socket-reuse-port 'false' Enable to set socket option SO_REUSEPORT on listeners allowing rebinding of a port already in use. --socket-reuse-address 'false' - Enable to set socket option SO_REUSEADDR on listeners allowing binding to an address in TIME_WAIT state. + Enable to set socket option SO_REUSEADDR on listeners allowing binding to an address in TIME_WAIT state. + --raft-read-timeout '` + rafthttp.DefaultConnReadTimeout.String() + `' + Read timeout set on each rafthttp connection + --raft-write-timeout '` + rafthttp.DefaultConnWriteTimeout.String() + `' + Write timeout set on each rafthttp connection Clustering: --initial-advertise-peer-urls 'http://localhost:2380' @@ -97,7 +108,8 @@ Clustering: --initial-cluster 'default=http://localhost:2380' Initial cluster configuration for bootstrapping. --initial-cluster-state 'new' - Initial cluster state ('new' or 'existing'). + Initial cluster state ('new' when bootstrapping a new cluster or 'existing' when adding new members to an existing cluster). + After successful initialization (bootstrapping or adding), flag is ignored on restarts. --initial-cluster-token 'etcd-cluster' Initial cluster token for the etcd cluster during bootstrap. Specifying this can protect you from unintended cross-cluster interaction when running multiple clusters. @@ -140,10 +152,14 @@ Security: Path to the client server TLS key file. --client-cert-auth 'false' Enable client cert authentication. + --client-cert-file '' + Path to an explicit peer client TLS cert file otherwise cert file will be used when client auth is required. + --client-key-file '' + Path to an explicit peer client TLS key file otherwise key file will be used when client auth is required. --client-crl-file '' Path to the client certificate revocation list file. --client-cert-allowed-hostname '' - Allowed TLS hostname for client cert authentication. + Comma-separated list of SAN hostnames for client cert authentication. --trusted-ca-file '' Path to the client server TLS trusted CA cert file. --auto-tls 'false' @@ -157,11 +173,15 @@ Security: --peer-trusted-ca-file '' Path to the peer server TLS trusted CA file. --peer-cert-allowed-cn '' - Required CN for client certs connecting to the peer endpoint. + Comma-separated list of allowed CNs for inter-peer TLS authentication. --peer-cert-allowed-hostname '' - Allowed TLS hostname for inter peer authentication. + Comma-separated list of allowed SAN hostnames for inter-peer TLS authentication. --peer-auto-tls 'false' Peer TLS using self-generated certificates if --peer-key-file and --peer-cert-file are not provided. + --peer-client-cert-file '' + Path to an explicit peer client TLS cert file otherwise peer cert file will be used when client auth is required. + --peer-client-key-file '' + Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required. --self-signed-cert-validity '1' The validity period of the client and peer certificates that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS, the unit is year, and the default is 1. --peer-crl-file '' @@ -172,6 +192,10 @@ Security: Comma-separated whitelist of origins for CORS, or cross-origin resource sharing, (empty or * means allow all). --host-whitelist '*' Acceptable hostnames from HTTP client requests, if server is not secure (empty or * means allow all). + --tls-min-version 'TLS1.2' + Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3. + --tls-max-version '' + Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go). Auth: --auth-token 'simple' @@ -187,7 +211,7 @@ Profiling and Monitoring: --metrics 'basic' Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics. --listen-metrics-urls '' - List of URLs to listen on for the metrics and health endpoints. + List of URLs to listen on for the /metrics and /health endpoints. For https, the client URL TLS info is used. Logging: --logger 'zap' @@ -210,8 +234,10 @@ Experimental distributed tracing: Distributed tracing service name, must be same across all etcd instances. --experimental-distributed-tracing-instance-id '' Distributed tracing instance ID, must be unique per each etcd instance. + --experimental-distributed-tracing-sampling-rate '0' + Number of samples to collect per million spans for distributed tracing. Disabled by default. -v2 Proxy (to be deprecated in v3.6): +v2 Proxy (Deprecated and to be decommissioned in v3.6): --proxy 'off' Proxy mode setting ('off', 'readonly' or 'on'). --proxy-failure-wait 5000 @@ -230,22 +256,38 @@ Experimental feature: Enable to check data corruption before serving any client/peer traffic. --experimental-corrupt-check-time '0s' Duration of time between cluster corruption check passes. + --experimental-compact-hash-check-enabled 'false' + Enable leader to periodically check followers compaction hashes. + --experimental-compact-hash-check-time '1m' + Duration of time between leader checks followers compaction hashes. --experimental-enable-v2v3 '' Serve v2 requests through the v3 backend under a given prefix. Deprecated and to be decommissioned in v3.6. --experimental-enable-lease-checkpoint 'false' ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. + --experimental-enable-lease-checkpoint-persist 'false' + Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled. --experimental-compaction-batch-limit 1000 ExperimentalCompactionBatchLimit sets the maximum revisions deleted in each compaction batch. + --experimental-compaction-sleep-interval '10ms' + ExperimentalCompactionSleepInterval sets the sleep interval between each compaction batch. --experimental-peer-skip-client-san-verification 'false' Skip verification of SAN field in client certificate for peer connections. --experimental-watch-progress-notify-interval '10m' Duration of periodical watch progress notification. + --experimental-downgrade-check-time + Duration of time between two downgrade status checks. + --experimental-memory-mlock + Enable to enforce etcd pages (in particular bbolt) to stay in RAM. --experimental-warning-apply-duration '100ms' - Warning is generated if requests take more than this duration. + Warning is generated if requests take more than this duration. --experimental-txn-mode-write-with-shared-buffer 'true' Enable the write transaction to use a shared buffer in its readonly check operations. --experimental-bootstrap-defrag-threshold-megabytes Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect. + --experimental-max-learners '1' + Set the max number of learner members allowed in the cluster membership. + --experimental-stop-grpc-service-on-defrag + Enable etcd gRPC service to stop serving client requests on defragmentation. Unsafe feature: --force-new-cluster 'false' diff --git a/server/etcdmain/main.go b/server/etcdmain/main.go index 2e67a137cc3b..e28e7da928d4 100644 --- a/server/etcdmain/main.go +++ b/server/etcdmain/main.go @@ -41,9 +41,6 @@ func Main(args []string) { } func notifySystemd(lg *zap.Logger) { - if lg == nil { - lg = zap.NewExample() - } lg.Info("notifying init daemon") _, err := daemon.SdNotify(false, daemon.SdNotifyReady) if err != nil { diff --git a/server/etcdserver/api/etcdhttp/health.go b/server/etcdserver/api/etcdhttp/health.go new file mode 100644 index 000000000000..fe22fb42c8a0 --- /dev/null +++ b/server/etcdserver/api/etcdhttp/health.go @@ -0,0 +1,472 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the http endpoints for etcd health checks. +// The endpoints include /livez, /readyz and /health. + +package etcdhttp + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "path" + "strings" + "time" + + "go.uber.org/zap" + + "github.com/prometheus/client_golang/prometheus" + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/server/v3/auth" + "go.etcd.io/etcd/server/v3/config" + "go.etcd.io/etcd/server/v3/etcdserver" +) + +const ( + PathHealth = "/health" + PathProxyHealth = "/proxy/health" + HealthStatusSuccess string = "success" + HealthStatusError string = "error" + checkTypeLivez = "livez" + checkTypeReadyz = "readyz" + checkTypeHealth = "health" +) + +type ServerHealth interface { + serverHealthV2V3 + Range(context.Context, *pb.RangeRequest) (*pb.RangeResponse, error) + Config() config.ServerConfig + AuthStore() auth.AuthStore + IsLearner() bool +} + +type serverHealthV2V3 interface { + Alarms() []*pb.AlarmMember + Leader() types.ID +} + +// HandleHealthForV2 registers metrics and health handlers for v2. +func HandleHealthForV2(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.ServerV2) { + mux.Handle(PathHealth, NewHealthHandler(lg, func(ctx context.Context, excludedAlarms StringSet, serializable bool) Health { + if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" { + return h + } + if h := checkLeader(lg, srv, serializable); h.Health != "true" { + return h + } + return checkV2API(ctx, lg, srv) + })) +} + +// HandleHealth registers metrics and health handlers. it checks health by using v3 range request +// and its corresponding timeout. +func HandleHealth(lg *zap.Logger, mux *http.ServeMux, srv ServerHealth) { + mux.Handle(PathHealth, NewHealthHandler(lg, func(ctx context.Context, excludedAlarms StringSet, serializable bool) Health { + if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" { + return h + } + if h := checkLeader(lg, srv, serializable); h.Health != "true" { + return h + } + return checkAPI(ctx, lg, srv, serializable) + })) + + installLivezEndpoints(lg, mux, srv) + installReadyzEndpoints(lg, mux, srv) +} + +// NewHealthHandler handles '/health' requests. +func NewHealthHandler(lg *zap.Logger, hfunc func(ctx context.Context, excludedAlarms StringSet, Serializable bool) Health) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + lg.Warn("/health error", zap.Int("status-code", http.StatusMethodNotAllowed)) + return + } + excludedAlarms := getQuerySet(r, "exclude") + // Passing the query parameter "serializable=true" ensures that the + // health of the local etcd is checked vs the health of the cluster. + // This is useful for probes attempting to validate the liveness of + // the etcd process vs readiness of the cluster to serve requests. + serializableFlag := getSerializableFlag(r) + h := hfunc(r.Context(), excludedAlarms, serializableFlag) + defer func() { + if h.Health == "true" { + healthSuccess.Inc() + } else { + healthFailed.Inc() + } + }() + d, _ := json.Marshal(h) + if h.Health != "true" { + http.Error(w, string(d), http.StatusServiceUnavailable) + lg.Warn("/health error", zap.String("output", string(d)), zap.Int("status-code", http.StatusServiceUnavailable)) + return + } + w.WriteHeader(http.StatusOK) + w.Write(d) + lg.Debug("/health OK", zap.Int("status-code", http.StatusOK)) + } +} + +var ( + healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "health_success", + Help: "The total number of successful health checks", + }) + healthFailed = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "health_failures", + Help: "The total number of failed health checks", + }) + healthCheckGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "healthcheck", + Help: "The result of each kind of healthcheck.", + }, + []string{"type", "name"}, + ) + healthCheckCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "healthchecks_total", + Help: "The total number of each kind of healthcheck.", + }, + []string{"type", "name", "status"}, + ) +) + +func init() { + prometheus.MustRegister(healthSuccess) + prometheus.MustRegister(healthFailed) + prometheus.MustRegister(healthCheckGauge) + prometheus.MustRegister(healthCheckCounter) +} + +// Health defines etcd server health status. +// TODO: remove manual parsing in etcdctl cluster-health +type Health struct { + Health string `json:"health"` + Reason string `json:"reason"` +} + +// HealthStatus is used in new /readyz or /livez health checks instead of the Health struct. +type HealthStatus struct { + Reason string `json:"reason"` + Status string `json:"status"` +} + +func getQuerySet(r *http.Request, query string) StringSet { + querySet := make(map[string]struct{}) + qs, found := r.URL.Query()[query] + if found { + for _, q := range qs { + if len(q) == 0 { + continue + } + querySet[q] = struct{}{} + } + } + return querySet +} + +func getSerializableFlag(r *http.Request) bool { + return r.URL.Query().Get("serializable") == "true" +} + +// TODO: etcdserver.ErrNoLeader in health API + +func checkAlarms(lg *zap.Logger, srv serverHealthV2V3, excludedAlarms StringSet) Health { + h := Health{Health: "true"} + as := srv.Alarms() + if len(as) > 0 { + for _, v := range as { + alarmName := v.Alarm.String() + if _, found := excludedAlarms[alarmName]; found { + lg.Debug("/health excluded alarm", zap.String("alarm", v.String())) + continue + } + + h.Health = "false" + switch v.Alarm { + case pb.AlarmType_NOSPACE: + h.Reason = "ALARM NOSPACE" + case pb.AlarmType_CORRUPT: + h.Reason = "ALARM CORRUPT" + default: + h.Reason = "ALARM UNKNOWN" + } + lg.Warn("serving /health false due to an alarm", zap.String("alarm", v.String())) + return h + } + } + + return h +} + +func checkLeader(lg *zap.Logger, srv serverHealthV2V3, serializable bool) Health { + h := Health{Health: "true"} + if !serializable && (uint64(srv.Leader()) == raft.None) { + h.Health = "false" + h.Reason = "RAFT NO LEADER" + lg.Warn("serving /health false; no leader") + } + return h +} + +func checkV2API(ctx context.Context, lg *zap.Logger, srv etcdserver.ServerV2) Health { + h := Health{Health: "true"} + ctx, cancel := context.WithTimeout(ctx, time.Second) + _, err := srv.Do(ctx, pb.Request{Method: "QGET"}) + cancel() + if err != nil { + h.Health = "false" + h.Reason = fmt.Sprintf("QGET ERROR:%s", err) + lg.Warn("serving /health false; QGET fails", zap.Error(err)) + return h + } + lg.Debug("serving /health true") + return h +} + +func checkAPI(ctx context.Context, lg *zap.Logger, srv ServerHealth, serializable bool) Health { + h := Health{Health: "true"} + cfg := srv.Config() + ctx = srv.AuthStore().WithRoot(ctx) + cctx, cancel := context.WithTimeout(ctx, cfg.ReqTimeout()) + _, err := srv.Range(cctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) + cancel() + if err != nil { + h.Health = "false" + h.Reason = fmt.Sprintf("RANGE ERROR:%s", err) + lg.Warn("serving /health false; Range fails", zap.Error(err)) + return h + } + lg.Debug("serving /health true") + return h +} + +type HealthCheck func(ctx context.Context) error + +type CheckRegistry struct { + checkType string + checks map[string]HealthCheck +} + +func installLivezEndpoints(lg *zap.Logger, mux *http.ServeMux, server ServerHealth) { + reg := CheckRegistry{checkType: checkTypeLivez, checks: make(map[string]HealthCheck)} + reg.Register("serializable_read", readCheck(server, true /* serializable */)) + reg.InstallHttpEndpoints(lg, mux) +} + +func installReadyzEndpoints(lg *zap.Logger, mux *http.ServeMux, server ServerHealth) { + reg := CheckRegistry{checkType: checkTypeReadyz, checks: make(map[string]HealthCheck)} + reg.Register("data_corruption", activeAlarmCheck(server, pb.AlarmType_CORRUPT)) + // serializable_read checks if local read is ok. + // linearizable_read checks if there is consensus in the cluster. + // Having both serializable_read and linearizable_read helps isolate the cause of problems if there is a read failure. + reg.Register("serializable_read", readCheck(server, true)) + // linearizable_read check would be replaced by read_index check in 3.6 + reg.Register("linearizable_read", readCheck(server, false)) + // check if local is learner + reg.Register("non_learner", learnerCheck(server)) + reg.InstallHttpEndpoints(lg, mux) +} + +func (reg *CheckRegistry) Register(name string, check HealthCheck) { + reg.checks[name] = check +} + +func (reg *CheckRegistry) RootPath() string { + return "/" + reg.checkType +} + +func (reg *CheckRegistry) InstallHttpEndpoints(lg *zap.Logger, mux *http.ServeMux) { + checkNames := make([]string, 0, len(reg.checks)) + for k := range reg.checks { + checkNames = append(checkNames, k) + } + + // installs the http handler for the root path. + reg.installRootHttpEndpoint(lg, mux, checkNames...) + for _, checkName := range checkNames { + // installs the http handler for the individual check sub path. + subpath := path.Join(reg.RootPath(), checkName) + check := checkName + mux.Handle(subpath, newHealthHandler(subpath, lg, func(r *http.Request) HealthStatus { + return reg.runHealthChecks(r.Context(), check) + })) + } +} + +func (reg *CheckRegistry) runHealthChecks(ctx context.Context, checkNames ...string) HealthStatus { + h := HealthStatus{Status: HealthStatusSuccess} + var individualCheckOutput bytes.Buffer + for _, checkName := range checkNames { + check, found := reg.checks[checkName] + if !found { + panic(fmt.Errorf("Health check: %s not registered", checkName)) + } + if err := check(ctx); err != nil { + fmt.Fprintf(&individualCheckOutput, "[-]%s failed: %v\n", checkName, err) + h.Status = HealthStatusError + recordMetrics(reg.checkType, checkName, HealthStatusError) + } else { + fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", checkName) + recordMetrics(reg.checkType, checkName, HealthStatusSuccess) + } + } + h.Reason = individualCheckOutput.String() + return h +} + +// installRootHttpEndpoint installs the http handler for the root path. +func (reg *CheckRegistry) installRootHttpEndpoint(lg *zap.Logger, mux *http.ServeMux, checks ...string) { + hfunc := func(r *http.Request) HealthStatus { + // extracts the health check names to be excludeList from the query param + excluded := getQuerySet(r, "exclude") + + filteredCheckNames := filterCheckList(lg, listToStringSet(checks), excluded) + h := reg.runHealthChecks(r.Context(), filteredCheckNames...) + return h + } + mux.Handle(reg.RootPath(), newHealthHandler(reg.RootPath(), lg, hfunc)) +} + +// newHealthHandler generates a http HandlerFunc for a health check function hfunc. +func newHealthHandler(path string, lg *zap.Logger, hfunc func(*http.Request) HealthStatus) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + lg.Warn("Health request error", zap.String("path", path), zap.Int("status-code", http.StatusMethodNotAllowed)) + return + } + h := hfunc(r) + // Always returns detailed reason for failed checks. + if h.Status == HealthStatusError { + http.Error(w, h.Reason, http.StatusServiceUnavailable) + lg.Error("Health check error", zap.String("path", path), zap.String("reason", h.Reason), zap.Int("status-code", http.StatusServiceUnavailable)) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + // Only writes detailed reason for verbose requests. + if _, found := r.URL.Query()["verbose"]; found { + fmt.Fprint(w, h.Reason) + } + fmt.Fprint(w, "ok\n") + lg.Debug("Health check OK", zap.String("path", path), zap.String("reason", h.Reason), zap.Int("status-code", http.StatusOK)) + } +} + +func filterCheckList(lg *zap.Logger, checks StringSet, excluded StringSet) []string { + filteredList := []string{} + for chk := range checks { + if _, found := excluded[chk]; found { + delete(excluded, chk) + continue + } + filteredList = append(filteredList, chk) + } + if len(excluded) > 0 { + // For version compatibility, excluding non-exist checks would not fail the request. + lg.Warn("some health checks cannot be excluded", zap.String("missing-health-checks", formatQuoted(excluded.List()...))) + } + return filteredList +} + +// formatQuoted returns a formatted string of the health check names, +// preserving the order passed in. +func formatQuoted(names ...string) string { + quoted := make([]string, 0, len(names)) + for _, name := range names { + quoted = append(quoted, fmt.Sprintf("%q", name)) + } + return strings.Join(quoted, ",") +} + +type StringSet map[string]struct{} + +func (s StringSet) List() []string { + keys := make([]string, 0, len(s)) + for k := range s { + keys = append(keys, k) + } + return keys +} + +func listToStringSet(list []string) StringSet { + set := make(map[string]struct{}) + for _, s := range list { + set[s] = struct{}{} + } + return set +} + +func recordMetrics(checkType, name string, status string) { + val := 0.0 + if status == HealthStatusSuccess { + val = 1.0 + } + healthCheckGauge.With(prometheus.Labels{ + "type": checkType, + "name": name, + }).Set(val) + healthCheckCounter.With(prometheus.Labels{ + "type": checkType, + "name": name, + "status": status, + }).Inc() +} + +// activeAlarmCheck checks if a specific alarm type is active in the server. +func activeAlarmCheck(srv ServerHealth, at pb.AlarmType) func(context.Context) error { + return func(ctx context.Context) error { + as := srv.Alarms() + for _, v := range as { + if v.Alarm == at { + return fmt.Errorf("alarm activated: %s", at.String()) + } + } + return nil + } +} + +func readCheck(srv ServerHealth, serializable bool) func(ctx context.Context) error { + return func(ctx context.Context) error { + ctx = srv.AuthStore().WithRoot(ctx) + _, err := srv.Range(ctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) + return err + } +} + +func learnerCheck(srv ServerHealth) func(ctx context.Context) error { + return func(ctx context.Context) error { + if srv.IsLearner() { + return fmt.Errorf("not supported for learner") + } + return nil + } +} diff --git a/server/etcdserver/api/etcdhttp/health_test.go b/server/etcdserver/api/etcdhttp/health_test.go new file mode 100644 index 000000000000..fae41761d84d --- /dev/null +++ b/server/etcdserver/api/etcdhttp/health_test.go @@ -0,0 +1,456 @@ +package etcdhttp + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap/zaptest" + + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/testutil" + "go.etcd.io/etcd/client/pkg/v3/types" + "go.etcd.io/etcd/raft/v3" + "go.etcd.io/etcd/server/v3/auth" + "go.etcd.io/etcd/server/v3/config" + betesting "go.etcd.io/etcd/server/v3/mvcc/backend/testing" +) + +type fakeHealthServer struct { + fakeServer + serializableReadError error + linearizableReadError error + missingLeader bool + authStore auth.AuthStore + isLearner bool +} + +func (s *fakeHealthServer) Range(_ context.Context, req *pb.RangeRequest) (*pb.RangeResponse, error) { + if req.Serializable { + return nil, s.serializableReadError + } + return nil, s.linearizableReadError +} + +func (s *fakeHealthServer) IsLearner() bool { + return s.isLearner +} + +func (s *fakeHealthServer) Config() config.ServerConfig { + return config.ServerConfig{} +} + +func (s *fakeHealthServer) Leader() types.ID { + if !s.missingLeader { + return 1 + } + return types.ID(raft.None) +} + +func (s *fakeHealthServer) AuthStore() auth.AuthStore { return s.authStore } + +func (s *fakeHealthServer) ClientCertAuthEnabled() bool { return false } + +type healthTestCase struct { + name string + healthCheckURL string + expectStatusCode int + inResult []string + notInResult []string + + alarms []*pb.AlarmMember + apiError error + missingLeader bool + isLearner bool +} + +func TestHealthHandler(t *testing.T) { + // define the input and expected output + // input: alarms, and healthCheckURL + tests := []healthTestCase{ + { + name: "Healthy if no alarm", + alarms: []*pb.AlarmMember{}, + healthCheckURL: "/health", + expectStatusCode: http.StatusOK, + }, + { + name: "Unhealthy if NOSPACE alarm is on", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, + healthCheckURL: "/health", + expectStatusCode: http.StatusServiceUnavailable, + }, + { + name: "Healthy if NOSPACE alarm is on and excluded", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, + healthCheckURL: "/health?exclude=NOSPACE", + expectStatusCode: http.StatusOK, + }, + { + name: "Healthy if NOSPACE alarm is excluded", + alarms: []*pb.AlarmMember{}, + healthCheckURL: "/health?exclude=NOSPACE", + expectStatusCode: http.StatusOK, + }, + { + name: "Healthy if multiple NOSPACE alarms are on and excluded", + alarms: []*pb.AlarmMember{{MemberID: uint64(1), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(2), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(3), Alarm: pb.AlarmType_NOSPACE}}, + healthCheckURL: "/health?exclude=NOSPACE", + expectStatusCode: http.StatusOK, + }, + { + name: "Unhealthy if NOSPACE alarms is excluded and CORRUPT is on", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}}, + healthCheckURL: "/health?exclude=NOSPACE", + expectStatusCode: http.StatusServiceUnavailable, + }, + { + name: "Unhealthy if both NOSPACE and CORRUPT are on and excluded", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}}, + healthCheckURL: "/health?exclude=NOSPACE&exclude=CORRUPT", + expectStatusCode: http.StatusOK, + }, + { + name: "Unhealthy if api is not available", + healthCheckURL: "/health", + apiError: fmt.Errorf("Unexpected error"), + expectStatusCode: http.StatusServiceUnavailable, + }, + { + name: "Unhealthy if no leader", + healthCheckURL: "/health", + expectStatusCode: http.StatusServiceUnavailable, + missingLeader: true, + }, + { + name: "Healthy if no leader and serializable=true", + healthCheckURL: "/health?serializable=true", + expectStatusCode: http.StatusOK, + missingLeader: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + lg := zaptest.NewLogger(t) + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + HandleHealth(zaptest.NewLogger(t), mux, &fakeHealthServer{ + fakeServer: fakeServer{alarms: tt.alarms}, + serializableReadError: tt.apiError, + linearizableReadError: tt.apiError, + missingLeader: tt.missingLeader, + authStore: auth.NewAuthStore(lg, be, nil, 0), + }) + ts := httptest.NewServer(mux) + defer ts.Close() + checkHttpResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, nil, nil) + }) + } +} + +func TestHttpSubPath(t *testing.T) { + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + tests := []healthTestCase{ + { + name: "/readyz/data_corruption ok", + healthCheckURL: "/readyz/data_corruption", + expectStatusCode: http.StatusOK, + }, + { + name: "/readyz/serializable_read not ok with error", + apiError: fmt.Errorf("Unexpected error"), + healthCheckURL: "/readyz/serializable_read", + expectStatusCode: http.StatusServiceUnavailable, + notInResult: []string{"data_corruption"}, + }, + { + name: "/readyz/learner ok", + healthCheckURL: "/readyz/non_learner", + expectStatusCode: http.StatusOK, + }, + { + name: "/readyz/non_exist 404", + healthCheckURL: "/readyz/non_exist", + expectStatusCode: http.StatusNotFound, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + logger := zaptest.NewLogger(t) + s := &fakeHealthServer{ + serializableReadError: tt.apiError, + authStore: auth.NewAuthStore(logger, be, nil, 0), + } + HandleHealth(logger, mux, s) + ts := httptest.NewServer(mux) + defer ts.Close() + checkHttpResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) + checkMetrics(t, tt.healthCheckURL, "", tt.expectStatusCode) + }) + } +} + +func TestDataCorruptionCheck(t *testing.T) { + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + tests := []healthTestCase{ + { + name: "Live if CORRUPT alarm is on", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}}, + healthCheckURL: "/livez", + expectStatusCode: http.StatusOK, + notInResult: []string{"data_corruption"}, + }, + { + name: "Not ready if CORRUPT alarm is on", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}}, + healthCheckURL: "/readyz", + expectStatusCode: http.StatusServiceUnavailable, + inResult: []string{"[-]data_corruption failed: alarm activated: CORRUPT"}, + }, + { + name: "ready if CORRUPT alarm is not on", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, + healthCheckURL: "/readyz", + expectStatusCode: http.StatusOK, + }, + { + name: "ready if CORRUPT alarm is excluded", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}, {MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, + healthCheckURL: "/readyz?exclude=data_corruption", + expectStatusCode: http.StatusOK, + }, + { + name: "Not ready if CORRUPT alarm is on", + alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}}, + healthCheckURL: "/readyz?exclude=non_exist", + expectStatusCode: http.StatusServiceUnavailable, + inResult: []string{"[-]data_corruption failed: alarm activated: CORRUPT"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + logger := zaptest.NewLogger(t) + s := &fakeHealthServer{ + authStore: auth.NewAuthStore(logger, be, nil, 0), + } + HandleHealth(logger, mux, s) + ts := httptest.NewServer(mux) + defer ts.Close() + // OK before alarms are activated. + checkHttpResponse(t, ts, tt.healthCheckURL, http.StatusOK, nil, nil) + // Activate the alarms. + s.alarms = tt.alarms + checkHttpResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) + }) + } +} + +func TestSerializableReadCheck(t *testing.T) { + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + tests := []healthTestCase{ + { + name: "Alive normal", + healthCheckURL: "/livez?verbose", + expectStatusCode: http.StatusOK, + inResult: []string{"[+]serializable_read ok"}, + }, + { + name: "Not alive if range api is not available", + healthCheckURL: "/livez", + apiError: fmt.Errorf("Unexpected error"), + expectStatusCode: http.StatusServiceUnavailable, + inResult: []string{"[-]serializable_read failed: Unexpected error"}, + }, + { + name: "Not ready if range api is not available", + healthCheckURL: "/readyz", + apiError: fmt.Errorf("Unexpected error"), + expectStatusCode: http.StatusServiceUnavailable, + inResult: []string{"[-]serializable_read failed: Unexpected error"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + logger := zaptest.NewLogger(t) + s := &fakeHealthServer{ + serializableReadError: tt.apiError, + authStore: auth.NewAuthStore(logger, be, nil, 0), + } + HandleHealth(logger, mux, s) + ts := httptest.NewServer(mux) + defer ts.Close() + checkHttpResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) + checkMetrics(t, tt.healthCheckURL, "serializable_read", tt.expectStatusCode) + }) + } +} + +func TestLinearizableReadCheck(t *testing.T) { + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + tests := []healthTestCase{ + { + name: "Alive normal", + healthCheckURL: "/livez?verbose", + expectStatusCode: http.StatusOK, + inResult: []string{"[+]serializable_read ok"}, + }, + { + name: "Alive if lineariable range api is not available", + healthCheckURL: "/livez", + apiError: fmt.Errorf("Unexpected error"), + expectStatusCode: http.StatusOK, + }, + { + name: "Not ready if range api is not available", + healthCheckURL: "/readyz", + apiError: fmt.Errorf("Unexpected error"), + expectStatusCode: http.StatusServiceUnavailable, + inResult: []string{"[+]serializable_read ok", "[-]linearizable_read failed: Unexpected error"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + logger := zaptest.NewLogger(t) + s := &fakeHealthServer{ + linearizableReadError: tt.apiError, + authStore: auth.NewAuthStore(logger, be, nil, 0), + } + HandleHealth(logger, mux, s) + ts := httptest.NewServer(mux) + defer ts.Close() + checkHttpResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) + checkMetrics(t, tt.healthCheckURL, "linearizable_read", tt.expectStatusCode) + }) + } +} + +func TestLearnerReadyCheck(t *testing.T) { + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + tests := []healthTestCase{ + { + name: "readyz normal", + healthCheckURL: "/readyz", + expectStatusCode: http.StatusOK, + isLearner: false, + }, + { + name: "not ready because member is learner", + healthCheckURL: "/readyz", + expectStatusCode: http.StatusServiceUnavailable, + isLearner: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux := http.NewServeMux() + logger := zaptest.NewLogger(t) + s := &fakeHealthServer{ + linearizableReadError: tt.apiError, + authStore: auth.NewAuthStore(logger, be, nil, 0), + } + s.isLearner = tt.isLearner + HandleHealth(logger, mux, s) + ts := httptest.NewServer(mux) + defer ts.Close() + checkHttpResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) + checkMetrics(t, tt.healthCheckURL, "linearizable_read", tt.expectStatusCode) + }) + } +} + +func checkHttpResponse(t *testing.T, ts *httptest.Server, url string, expectStatusCode int, inResult []string, notInResult []string) { + res, err := ts.Client().Do(&http.Request{Method: http.MethodGet, URL: testutil.MustNewURL(t, ts.URL+url)}) + + if err != nil { + t.Fatalf("fail serve http request %s %v", url, err) + } + if res.StatusCode != expectStatusCode { + t.Errorf("want statusCode %d but got %d", expectStatusCode, res.StatusCode) + } + defer res.Body.Close() + b, err := io.ReadAll(res.Body) + if err != nil { + t.Fatalf("Failed to read response for %s", url) + } + result := string(b) + for _, substr := range inResult { + if !strings.Contains(result, substr) { + t.Errorf("Could not find substring : %s, in response: %s", substr, result) + return + } + } + for _, substr := range notInResult { + if strings.Contains(result, substr) { + t.Errorf("Do not expect substring : %s, in response: %s", substr, result) + return + } + } +} + +func checkMetrics(t *testing.T, url, checkName string, expectStatusCode int) { + defer healthCheckGauge.Reset() + defer healthCheckCounter.Reset() + + typeName := strings.TrimPrefix(strings.Split(url, "?")[0], "/") + if len(checkName) == 0 { + checkName = strings.Split(typeName, "/")[1] + typeName = strings.Split(typeName, "/")[0] + } + + expectedSuccessCount := 1 + expectedErrorCount := 0 + if expectStatusCode != http.StatusOK { + expectedSuccessCount = 0 + expectedErrorCount = 1 + } + + gather, _ := prometheus.DefaultGatherer.Gather() + for _, mf := range gather { + name := *mf.Name + val := 0 + switch name { + case "etcd_server_healthcheck": + val = int(mf.GetMetric()[0].GetGauge().GetValue()) + case "etcd_server_healthcheck_total": + val = int(mf.GetMetric()[0].GetCounter().GetValue()) + default: + continue + } + labelMap := make(map[string]string) + for _, label := range mf.GetMetric()[0].Label { + labelMap[label.GetName()] = label.GetValue() + } + if typeName != labelMap["type"] { + continue + } + if labelMap["name"] != checkName { + continue + } + if statusLabel, found := labelMap["status"]; found && statusLabel == HealthStatusError { + if val != expectedErrorCount { + t.Fatalf("%s got errorCount %d, wanted %d\n", name, val, expectedErrorCount) + } + } else { + if val != expectedSuccessCount { + t.Fatalf("%s got expectedSuccessCount %d, wanted %d\n", name, val, expectedSuccessCount) + } + } + } +} diff --git a/server/etcdserver/api/etcdhttp/metrics.go b/server/etcdserver/api/etcdhttp/metrics.go index b14a13c9c5e7..bf7d4a4a445c 100644 --- a/server/etcdserver/api/etcdhttp/metrics.go +++ b/server/etcdserver/api/etcdhttp/metrics.go @@ -15,190 +15,17 @@ package etcdhttp import ( - "context" - "encoding/json" - "fmt" "net/http" - "time" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "go.etcd.io/etcd/api/v3/etcdserverpb" - "go.etcd.io/etcd/raft/v3" - "go.etcd.io/etcd/server/v3/etcdserver" - "go.uber.org/zap" ) const ( PathMetrics = "/metrics" - PathHealth = "/health" PathProxyMetrics = "/proxy/metrics" - PathProxyHealth = "/proxy/health" ) -// HandleMetricsHealth registers metrics and health handlers. -func HandleMetricsHealth(lg *zap.Logger, mux *http.ServeMux, srv etcdserver.ServerV2) { +// HandleMetrics registers prometheus handler on '/metrics'. +func HandleMetrics(mux *http.ServeMux) { mux.Handle(PathMetrics, promhttp.Handler()) - mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet) Health { return checkV2Health(lg, srv, excludedAlarms) })) -} - -// HandleMetricsHealthForV3 registers metrics and health handlers. it checks health by using v3 range request -// and its corresponding timeout. -func HandleMetricsHealthForV3(lg *zap.Logger, mux *http.ServeMux, srv *etcdserver.EtcdServer) { - mux.Handle(PathMetrics, promhttp.Handler()) - mux.Handle(PathHealth, NewHealthHandler(lg, func(excludedAlarms AlarmSet) Health { return checkV3Health(lg, srv, excludedAlarms) })) -} - -// HandlePrometheus registers prometheus handler on '/metrics'. -func HandlePrometheus(mux *http.ServeMux) { - mux.Handle(PathMetrics, promhttp.Handler()) -} - -// NewHealthHandler handles '/health' requests. -func NewHealthHandler(lg *zap.Logger, hfunc func(excludedAlarms AlarmSet) Health) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - w.Header().Set("Allow", http.MethodGet) - http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) - lg.Warn("/health error", zap.Int("status-code", http.StatusMethodNotAllowed)) - return - } - excludedAlarms := getExcludedAlarms(r) - h := hfunc(excludedAlarms) - defer func() { - if h.Health == "true" { - healthSuccess.Inc() - } else { - healthFailed.Inc() - } - }() - d, _ := json.Marshal(h) - if h.Health != "true" { - http.Error(w, string(d), http.StatusServiceUnavailable) - lg.Warn("/health error", zap.String("output", string(d)), zap.Int("status-code", http.StatusServiceUnavailable)) - return - } - w.WriteHeader(http.StatusOK) - w.Write(d) - lg.Debug("/health OK", zap.Int("status-code", http.StatusOK)) - } -} - -var ( - healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "etcd", - Subsystem: "server", - Name: "health_success", - Help: "The total number of successful health checks", - }) - healthFailed = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "etcd", - Subsystem: "server", - Name: "health_failures", - Help: "The total number of failed health checks", - }) -) - -func init() { - prometheus.MustRegister(healthSuccess) - prometheus.MustRegister(healthFailed) -} - -// Health defines etcd server health status. -// TODO: remove manual parsing in etcdctl cluster-health -type Health struct { - Health string `json:"health"` - Reason string `json:"reason"` -} - -type AlarmSet map[string]struct{} - -func getExcludedAlarms(r *http.Request) (alarms AlarmSet) { - alarms = make(map[string]struct{}, 2) - alms, found := r.URL.Query()["exclude"] - if found { - for _, alm := range alms { - if len(alms) == 0 { - continue - } - alarms[alm] = struct{}{} - } - } - return alarms -} - -// TODO: etcdserver.ErrNoLeader in health API - -func checkHealth(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSet) Health { - h := Health{} - h.Health = "true" - as := srv.Alarms() - if len(as) > 0 { - for _, v := range as { - alarmName := v.Alarm.String() - if _, found := excludedAlarms[alarmName]; found { - lg.Debug("/health excluded alarm", zap.String("alarm", alarmName)) - delete(excludedAlarms, alarmName) - continue - } - - h.Health = "false" - switch v.Alarm { - case etcdserverpb.AlarmType_NOSPACE: - h.Reason = "ALARM NOSPACE" - case etcdserverpb.AlarmType_CORRUPT: - h.Reason = "ALARM CORRUPT" - default: - h.Reason = "ALARM UNKNOWN" - } - lg.Warn("serving /health false due to an alarm", zap.String("alarm", v.String())) - return h - } - } - - if len(excludedAlarms) > 0 { - lg.Warn("fail exclude alarms from health check", zap.String("exclude alarms", fmt.Sprintf("%+v", excludedAlarms))) - } - - if uint64(srv.Leader()) == raft.None { - h.Health = "false" - h.Reason = "RAFT NO LEADER" - lg.Warn("serving /health false; no leader") - return h - } - return h -} - -func checkV2Health(lg *zap.Logger, srv etcdserver.ServerV2, excludedAlarms AlarmSet) (h Health) { - if h = checkHealth(lg, srv, excludedAlarms); h.Health != "true" { - return - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _, err := srv.Do(ctx, etcdserverpb.Request{Method: "QGET"}) - cancel() - if err != nil { - h.Health = "false" - h.Reason = fmt.Sprintf("QGET ERROR:%s", err) - lg.Warn("serving /health false; QGET fails", zap.Error(err)) - return - } - lg.Debug("serving /health true") - return -} - -func checkV3Health(lg *zap.Logger, srv *etcdserver.EtcdServer, excludedAlarms AlarmSet) (h Health) { - if h = checkHealth(lg, srv, excludedAlarms); h.Health != "true" { - return - } - ctx, cancel := context.WithTimeout(context.Background(), srv.Cfg.ReqTimeout()) - _, err := srv.Range(ctx, &etcdserverpb.RangeRequest{KeysOnly: true, Limit: 1}) - cancel() - if err != nil { - h.Health = "false" - h.Reason = fmt.Sprintf("RANGE ERROR:%s", err) - lg.Warn("serving /health false; Range fails", zap.Error(err)) - return - } - lg.Debug("serving /health true") - return } diff --git a/server/etcdserver/api/etcdhttp/metrics_test.go b/server/etcdserver/api/etcdhttp/metrics_test.go deleted file mode 100644 index c8839d7ad351..000000000000 --- a/server/etcdserver/api/etcdhttp/metrics_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package etcdhttp - -import ( - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - pb "go.etcd.io/etcd/api/v3/etcdserverpb" - "go.etcd.io/etcd/client/pkg/v3/testutil" - "go.etcd.io/etcd/client/pkg/v3/types" - "go.etcd.io/etcd/raft/v3" - "go.etcd.io/etcd/server/v3/etcdserver" - stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" - "go.uber.org/zap" -) - -type fakeStats struct{} - -func (s *fakeStats) SelfStats() []byte { return nil } -func (s *fakeStats) LeaderStats() []byte { return nil } -func (s *fakeStats) StoreStats() []byte { return nil } - -type fakeServerV2 struct { - fakeServer - stats.Stats - health string -} - -func (s *fakeServerV2) Leader() types.ID { - if s.health == "true" { - return 1 - } - return types.ID(raft.None) -} -func (s *fakeServerV2) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) { - if s.health == "true" { - return etcdserver.Response{}, nil - } - return etcdserver.Response{}, fmt.Errorf("fail health check") -} -func (s *fakeServerV2) ClientCertAuthEnabled() bool { return false } - -func TestHealthHandler(t *testing.T) { - // define the input and expected output - // input: alarms, and healthCheckURL - tests := []struct { - alarms []*pb.AlarmMember - healthCheckURL string - statusCode int - health string - }{ - { - []*pb.AlarmMember{}, - "/health", - http.StatusOK, - "true", - }, - { - []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, - "/health", - http.StatusServiceUnavailable, - "false", - }, - { - []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, - "/health?exclude=NOSPACE", - http.StatusOK, - "true", - }, - { - []*pb.AlarmMember{}, - "/health?exclude=NOSPACE", - http.StatusOK, - "true", - }, - { - []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}}, - "/health?exclude=NOSPACE", - http.StatusServiceUnavailable, - "false", - }, - { - []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}}, - "/health?exclude=NOSPACE&exclude=CORRUPT", - http.StatusOK, - "true", - }, - } - - for i, tt := range tests { - func() { - mux := http.NewServeMux() - HandleMetricsHealth(zap.NewExample(), mux, &fakeServerV2{ - fakeServer: fakeServer{alarms: tt.alarms}, - Stats: &fakeStats{}, - health: tt.health, - }) - ts := httptest.NewServer(mux) - defer ts.Close() - - res, err := ts.Client().Do(&http.Request{Method: http.MethodGet, URL: testutil.MustNewURL(t, ts.URL+tt.healthCheckURL)}) - if err != nil { - t.Errorf("fail serve http request %s %v in test case #%d", tt.healthCheckURL, err, i+1) - } - if res == nil { - t.Errorf("got nil http response with http request %s in test case #%d", tt.healthCheckURL, i+1) - return - } - if res.StatusCode != tt.statusCode { - t.Errorf("want statusCode %d but got %d in test case #%d", tt.statusCode, res.StatusCode, i+1) - } - health, err := parseHealthOutput(res.Body) - if err != nil { - t.Errorf("fail parse health check output %v", err) - } - if health.Health != tt.health { - t.Errorf("want health %s but got %s", tt.health, health.Health) - } - }() - } -} - -func parseHealthOutput(body io.Reader) (Health, error) { - obj := Health{} - d, derr := ioutil.ReadAll(body) - if derr != nil { - return obj, derr - } - if err := json.Unmarshal(d, &obj); err != nil { - return obj, err - } - return obj, nil -} diff --git a/server/etcdserver/api/membership/cluster.go b/server/etcdserver/api/membership/cluster.go index 3187d12f7d1f..38e0c1630ea9 100644 --- a/server/etcdserver/api/membership/cluster.go +++ b/server/etcdserver/api/membership/cluster.go @@ -20,6 +20,7 @@ import ( "crypto/sha1" "encoding/binary" "encoding/json" + "errors" "fmt" "path" "sort" @@ -32,16 +33,16 @@ import ( "go.etcd.io/etcd/pkg/v3/netutil" "go.etcd.io/etcd/raft/v3" "go.etcd.io/etcd/raft/v3/raftpb" + "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/mvcc/buckets" "github.com/coreos/go-semver/semver" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) -const maxLearners = 1 - // RaftCluster is a list of Members that belong to the same raft cluster type RaftCluster struct { lg *zap.Logger @@ -60,6 +61,7 @@ type RaftCluster struct { removed map[types.ID]bool downgradeInfo *DowngradeInfo + maxLearners int } // ConfigChangeContext represents a context for confChange. @@ -80,8 +82,8 @@ const ( // NewClusterFromURLsMap creates a new raft cluster using provided urls map. Currently, it does not support creating // cluster with raft learner member. -func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap) (*RaftCluster, error) { - c := NewCluster(lg) +func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap, opts ...ClusterOption) (*RaftCluster, error) { + c := NewCluster(lg, opts...) for name, urls := range urlsmap { m := NewMember(name, urls, token, nil) if _, ok := c.members[m.ID]; ok { @@ -96,8 +98,8 @@ func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap) return c, nil } -func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member) *RaftCluster { - c := NewCluster(lg) +func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member, opts ...ClusterOption) *RaftCluster { + c := NewCluster(lg, opts...) c.cid = id for _, m := range membs { c.members[m.ID] = m @@ -105,15 +107,18 @@ func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member) *RaftCl return c } -func NewCluster(lg *zap.Logger) *RaftCluster { +func NewCluster(lg *zap.Logger, opts ...ClusterOption) *RaftCluster { if lg == nil { lg = zap.NewNop() } + clOpts := newClusterOpts(opts...) + return &RaftCluster{ lg: lg, members: make(map[types.ID]*Member), removed: make(map[types.ID]bool), downgradeInfo: &DowngradeInfo{Enabled: false}, + maxLearners: clOpts.maxLearners, } } @@ -253,12 +258,12 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) { c.Lock() defer c.Unlock() - if c.be != nil { - c.version = clusterVersionFromBackend(c.lg, c.be) - c.members, c.removed = membersFromBackend(c.lg, c.be) - } else { + if c.v2store != nil { c.version = clusterVersionFromStore(c.lg, c.v2store) c.members, c.removed = membersFromStore(c.lg, c.v2store) + } else { + c.version = clusterVersionFromBackend(c.lg, c.be) + c.members, c.removed = membersFromBackend(c.lg, c.be) } if c.be != nil { @@ -278,6 +283,7 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) { zap.String("local-member-id", c.localID.String()), zap.String("recovered-remote-peer-id", m.ID.String()), zap.Strings("recovered-remote-peer-urls", m.PeerURLs), + zap.Bool("recovered-remote-peer-is-learner", m.IsLearner), ) } if c.version != nil { @@ -292,9 +298,9 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) { // ensures that it is still valid. func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { // TODO: this must be switched to backend as well. - members, removed := membersFromStore(c.lg, c.v2store) + membersMap, removedMap := membersFromStore(c.lg, c.v2store) id := types.ID(cc.NodeID) - if removed[id] { + if removedMap[id] { return ErrIDRemoved } switch cc.Type { @@ -305,19 +311,21 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { } if confChangeContext.IsPromote { // promoting a learner member to voting member - if members[id] == nil { + if membersMap[id] == nil { return ErrIDNotFound } - if !members[id].IsLearner { + if !membersMap[id].IsLearner { return ErrMemberNotLearner } } else { // adding a new member - if members[id] != nil { + if membersMap[id] != nil { return ErrIDExists } + var members []*Member urls := make(map[string]bool) - for _, m := range members { + for _, m := range membersMap { + members = append(members, m) for _, u := range m.PeerURLs { urls[u] = true } @@ -328,29 +336,24 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { } } - if confChangeContext.Member.IsLearner { // the new member is a learner - numLearners := 0 - for _, m := range members { - if m.IsLearner { - numLearners++ - } - } - if numLearners+1 > maxLearners { - return ErrTooManyLearners + if confChangeContext.Member.RaftAttributes.IsLearner && cc.Type == raftpb.ConfChangeAddLearnerNode { // the new member is a learner + scaleUpLearners := true + if err := ValidateMaxLearnerConfig(c.maxLearners, members, scaleUpLearners); err != nil { + return err } } } case raftpb.ConfChangeRemoveNode: - if members[id] == nil { + if membersMap[id] == nil { return ErrIDNotFound } case raftpb.ConfChangeUpdateNode: - if members[id] == nil { + if membersMap[id] == nil { return ErrIDNotFound } urls := make(map[string]bool) - for _, m := range members { + for _, m := range membersMap { if m.ID == id { continue } @@ -380,11 +383,37 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() + + var v2Err, beErr error if c.v2store != nil { - mustSaveMemberToStore(c.lg, c.v2store, m) + v2Err = unsafeSaveMemberToStore(c.lg, c.v2store, m) + if v2Err != nil { + if e, ok := v2Err.(*v2error.Error); !ok || e.ErrorCode != v2error.EcodeNodeExist { + c.lg.Panic( + "failed to save member to store", + zap.String("member-id", m.ID.String()), + zap.Error(v2Err), + ) + } + } } if c.be != nil && shouldApplyV3 { - mustSaveMemberToBackend(c.lg, c.be, m) + beErr = unsafeSaveMemberToBackend(c.lg, c.be, m) + if beErr != nil && !errors.Is(beErr, errMemberAlreadyExist) { + c.lg.Panic( + "failed to save member to backend", + zap.String("member-id", m.ID.String()), + zap.Error(beErr), + ) + } + } + // Panic if both storeV2 and backend report member already exist. + if v2Err != nil && (beErr != nil || c.be == nil) { + c.lg.Panic( + "failed to save member to store", + zap.String("member-id", m.ID.String()), + zap.Error(v2Err), + ) } c.members[m.ID] = m @@ -395,6 +424,7 @@ func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) { zap.String("local-member-id", c.localID.String()), zap.String("added-peer-id", m.ID.String()), zap.Strings("added-peer-peer-urls", m.PeerURLs), + zap.Bool("added-peer-is-learner", m.IsLearner), ) } @@ -403,11 +433,36 @@ func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) { func (c *RaftCluster) RemoveMember(id types.ID, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() + var v2Err, beErr error if c.v2store != nil { - mustDeleteMemberFromStore(c.lg, c.v2store, id) + v2Err = unsafeDeleteMemberFromStore(c.v2store, id) + if v2Err != nil { + if e, ok := v2Err.(*v2error.Error); !ok || e.ErrorCode != v2error.EcodeKeyNotFound { + c.lg.Panic( + "failed to delete member from store", + zap.String("member-id", id.String()), + zap.Error(v2Err), + ) + } + } } if c.be != nil && shouldApplyV3 { - mustDeleteMemberFromBackend(c.be, id) + beErr = unsafeDeleteMemberFromBackend(c.be, id) + if beErr != nil && !errors.Is(beErr, errMemberNotFound) { + c.lg.Panic( + "failed to delete member from backend", + zap.String("member-id", id.String()), + zap.Error(beErr), + ) + } + } + // Panic if both storeV2 and backend report member not found. + if v2Err != nil && (beErr != nil || c.be == nil) { + c.lg.Panic( + "failed to delete member from store", + zap.String("member-id", id.String()), + zap.Error(v2Err), + ) } m, ok := c.members[id] @@ -421,6 +476,7 @@ func (c *RaftCluster) RemoveMember(id types.ID, shouldApplyV3 ShouldApplyV3) { zap.String("local-member-id", c.localID.String()), zap.String("removed-remote-peer-id", id.String()), zap.Strings("removed-remote-peer-urls", m.PeerURLs), + zap.Bool("removed-remote-peer-is-learner", m.IsLearner), ) } else { c.lg.Warn( @@ -442,7 +498,7 @@ func (c *RaftCluster) UpdateAttributes(id types.ID, attr Attributes, shouldApply mustUpdateMemberAttrInStore(c.lg, c.v2store, m) } if c.be != nil && shouldApplyV3 { - mustSaveMemberToBackend(c.lg, c.be, m) + unsafeSaveMemberToBackend(c.lg, c.be, m) } return } @@ -475,13 +531,14 @@ func (c *RaftCluster) PromoteMember(id types.ID, shouldApplyV3 ShouldApplyV3) { mustUpdateMemberInStore(c.lg, c.v2store, c.members[id]) } if c.be != nil && shouldApplyV3 { - mustSaveMemberToBackend(c.lg, c.be, c.members[id]) + unsafeSaveMemberToBackend(c.lg, c.be, c.members[id]) } c.lg.Info( "promote member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), + zap.String("promoted-member-id", id.String()), ) } @@ -494,7 +551,7 @@ func (c *RaftCluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes, mustUpdateMemberInStore(c.lg, c.v2store, c.members[id]) } if c.be != nil && shouldApplyV3 { - mustSaveMemberToBackend(c.lg, c.be, c.members[id]) + unsafeSaveMemberToBackend(c.lg, c.be, c.members[id]) } c.lg.Info( @@ -503,6 +560,7 @@ func (c *RaftCluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes, zap.String("local-member-id", c.localID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", raftAttr.PeerURLs), + zap.Bool("updated-remote-peer-is-learner", raftAttr.IsLearner), ) } @@ -639,6 +697,10 @@ func (c *RaftCluster) IsReadyToPromoteMember(id uint64) bool { return true } +func (c *RaftCluster) MembersFromStore() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromStore(c.lg, c.v2store) +} + func membersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, map[types.ID]bool) { members := make(map[types.ID]*Member) removed := make(map[types.ID]bool) @@ -675,6 +737,10 @@ func membersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, m return members, removed } +func (c *RaftCluster) MembersFromBackend() (map[types.ID]*Member, map[types.ID]bool) { + return membersFromBackend(c.lg, c.be) +} + func membersFromBackend(lg *zap.Logger, be backend.Backend) (map[types.ID]*Member, map[types.ID]bool) { return mustReadMembersFromBackend(lg, be) } @@ -700,7 +766,7 @@ func clusterVersionFromBackend(lg *zap.Logger, be backend.Backend) *semver.Versi tx := be.ReadTx() tx.RLock() defer tx.RUnlock() - keys, vals := tx.UnsafeRange(clusterBucketName, ckey, nil, 0) + keys, vals := tx.UnsafeRange(buckets.Cluster, ckey, nil, 0) if len(keys) == 0 { return nil } @@ -719,7 +785,7 @@ func downgradeInfoFromBackend(lg *zap.Logger, be backend.Backend) *DowngradeInfo tx := be.ReadTx() tx.Lock() defer tx.Unlock() - keys, vals := tx.UnsafeRange(clusterBucketName, dkey, nil, 0) + keys, vals := tx.UnsafeRange(buckets.Cluster, dkey, nil, 0) if len(keys) == 0 { return nil } @@ -846,6 +912,7 @@ func (c *RaftCluster) IsMemberExist(id types.ID) bool { c.Lock() defer c.Unlock() _, ok := c.members[id] + // gofail: var afterIsMemberExist struct{} return ok } @@ -869,7 +936,7 @@ func (c *RaftCluster) PushMembershipToStorage() { if c.be != nil { TrimMembershipFromBackend(c.lg, c.be) for _, m := range c.members { - mustSaveMemberToBackend(c.lg, c.be, m) + unsafeSaveMemberToBackend(c.lg, c.be, m) } } if c.v2store != nil { @@ -879,3 +946,24 @@ func (c *RaftCluster) PushMembershipToStorage() { } } } + +// ValidateMaxLearnerConfig verifies the existing learner members in the cluster membership and an optional N+1 learner +// scale up are not more than maxLearners. +func ValidateMaxLearnerConfig(maxLearners int, members []*Member, scaleUpLearners bool) error { + numLearners := 0 + for _, m := range members { + if m.IsLearner { + numLearners++ + } + } + // Validate config can accommodate scale up. + if scaleUpLearners { + numLearners++ + } + + if numLearners > maxLearners { + return ErrTooManyLearners + } + + return nil +} diff --git a/server/etcdserver/api/membership/cluster_opts.go b/server/etcdserver/api/membership/cluster_opts.go new file mode 100644 index 000000000000..204fbf04d2c5 --- /dev/null +++ b/server/etcdserver/api/membership/cluster_opts.go @@ -0,0 +1,43 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package membership + +const DefaultMaxLearners = 1 + +type ClusterOptions struct { + maxLearners int +} + +// ClusterOption are options which can be applied to the raft cluster. +type ClusterOption func(*ClusterOptions) + +func newClusterOpts(opts ...ClusterOption) *ClusterOptions { + clOpts := &ClusterOptions{} + clOpts.applyOpts(opts) + return clOpts +} + +func (co *ClusterOptions) applyOpts(opts []ClusterOption) { + for _, opt := range opts { + opt(co) + } +} + +// WithMaxLearners sets the maximum number of learners that can exist in the cluster membership. +func WithMaxLearners(max int) ClusterOption { + return func(co *ClusterOptions) { + co.maxLearners = max + } +} diff --git a/server/etcdserver/api/membership/cluster_test.go b/server/etcdserver/api/membership/cluster_test.go index 23d81fec1cac..3329d27f86c4 100644 --- a/server/etcdserver/api/membership/cluster_test.go +++ b/server/etcdserver/api/membership/cluster_test.go @@ -20,8 +20,12 @@ import ( "path" "reflect" "testing" + "time" "github.com/coreos/go-semver/semver" + "github.com/stretchr/testify/assert" + betesting "go.etcd.io/etcd/server/v3/mvcc/backend/testing" + "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/testutil" @@ -29,8 +33,6 @@ import ( "go.etcd.io/etcd/raft/v3/raftpb" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/mock/mockstore" - - "go.uber.org/zap" ) func TestClusterMember(t *testing.T) { @@ -279,10 +281,14 @@ func TestClusterValidateAndAssignIDs(t *testing.T) { } func TestClusterValidateConfigurationChange(t *testing.T) { - cl := NewCluster(zaptest.NewLogger(t)) + cl := NewCluster(zaptest.NewLogger(t), WithMaxLearners(1)) cl.SetStore(v2store.New()) for i := 1; i <= 4; i++ { - attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", i)}} + var isLearner bool + if i == 1 { + isLearner = true + } + attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", i)}, IsLearner: isLearner} cl.AddMember(&Member{ID: types.ID(i), RaftAttributes: attr}, true) } cl.RemoveMember(4, true) @@ -327,6 +333,17 @@ func TestClusterValidateConfigurationChange(t *testing.T) { t.Fatal(err) } + attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 7)}, IsLearner: true} + ctx7, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(7), RaftAttributes: attr}}) + if err != nil { + t.Fatal(err) + } + + attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}, IsLearner: true} + ctx8, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(1), RaftAttributes: attr}, IsPromote: true}) + if err != nil { + t.Fatal(err) + } tests := []struct { cc raftpb.ConfChange werr error @@ -424,6 +441,22 @@ func TestClusterValidateConfigurationChange(t *testing.T) { }, ErrIDNotFound, }, + { + raftpb.ConfChange{ + Type: raftpb.ConfChangeAddLearnerNode, + NodeID: 7, + Context: ctx7, + }, + ErrTooManyLearners, + }, + { + raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: 1, + Context: ctx8, + }, + nil, + }, } for i, tt := range tests { err := cl.ValidateConfigurationChange(tt.cc) @@ -1019,3 +1052,192 @@ func TestIsVersionChangable(t *testing.T) { }) } } + +func TestAddMemberSyncsBackendAndStoreV2(t *testing.T) { + now := time.Now() + alice := NewMember("", nil, "alice", &now) + + tcs := []struct { + name string + + storeV2Nil bool + backendNil bool + storeV2Members []*Member + backendMembers []*Member + + expectPanics bool + expectMembers map[types.ID]*Member + }{ + { + name: "Adding new member should succeed", + }, + { + name: "Adding member should succeed if it was only in storeV2", + storeV2Members: []*Member{alice}, + }, + { + name: "Adding member should succeed if it was only in backend", + backendMembers: []*Member{alice}, + }, + { + name: "Adding member should success if it exists in both", + storeV2Members: []*Member{alice}, + backendMembers: []*Member{alice}, + }, + { + name: "Adding member should fail if it exists in storeV2 and backend is nil", + storeV2Members: []*Member{alice}, + backendNil: true, + expectPanics: true, + }, + { + name: "Adding member should succeed if it exists in backend and storageV2 is nil", + storeV2Nil: true, + backendMembers: []*Member{alice}, + }, + { + name: "Adding new member should succeed if backend is nil", + storeV2Members: []*Member{}, + backendNil: true, + }, + { + name: "Adding new member should fail if storageV2 is nil", + storeV2Nil: true, + backendMembers: []*Member{}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + lg := zaptest.NewLogger(t) + be, _ := betesting.NewDefaultTmpBackend(t) + defer be.Close() + mustCreateBackendBuckets(be) + st := v2store.New() + for _, m := range tc.backendMembers { + unsafeSaveMemberToBackend(lg, be, m) + } + be.ForceCommit() + for _, m := range tc.storeV2Members { + mustSaveMemberToStore(lg, st, m) + } + cluster := NewCluster(lg) + if !tc.backendNil { + cluster.SetBackend(be) + } + if !tc.storeV2Nil { + cluster.SetStore(st) + } + if tc.expectPanics { + assert.Panics(t, func() { + cluster.AddMember(alice, ApplyBoth) + }) + } else { + cluster.AddMember(alice, ApplyBoth) + } + if !tc.storeV2Nil { + storeV2Members, _ := membersFromStore(lg, st) + assert.Equal(t, map[types.ID]*Member{alice.ID: alice}, storeV2Members) + } + if !tc.backendNil { + be.ForceCommit() + beMembers, _ := mustReadMembersFromBackend(lg, be) + assert.Equal(t, map[types.ID]*Member{alice.ID: alice}, beMembers) + } + }) + } +} + +func TestRemoveMemberSyncsBackendAndStoreV2(t *testing.T) { + now := time.Now() + alice := NewMember("", nil, "alice", &now) + + tcs := []struct { + name string + + storeV2Nil bool + backendNil bool + storeV2Members []*Member + backendMembers []*Member + + expectMembers []*Member + expectPanics bool + }{ + { + name: "Removing new member should fail", + expectPanics: true, + }, + { + name: "Removing member should succeed if it was only in storeV2", + storeV2Members: []*Member{alice}, + }, + { + name: "Removing member should succeed if it was only in backend", + backendMembers: []*Member{alice}, + }, + { + name: "Removing member should succeed if it exists in both", + storeV2Members: []*Member{alice}, + backendMembers: []*Member{alice}, + }, + { + name: "Removing new member should fail if backend is nil", + storeV2Members: []*Member{}, + backendNil: true, + expectPanics: true, + }, + { + name: "Removing new member should succeed if storageV2 is nil", + storeV2Nil: true, + backendMembers: []*Member{}, + }, + { + name: "Removing member should succeed if it exists in v2storage and backend is nil", + storeV2Members: []*Member{alice}, + backendNil: true, + }, + { + name: "Removing member should succeed if it exists in backend and storageV2 is nil", + storeV2Nil: true, + backendMembers: []*Member{alice}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + lg := zaptest.NewLogger(t) + be, _ := betesting.NewDefaultTmpBackend(t) + defer be.Close() + mustCreateBackendBuckets(be) + st := v2store.New() + for _, m := range tc.backendMembers { + unsafeSaveMemberToBackend(lg, be, m) + } + be.ForceCommit() + for _, m := range tc.storeV2Members { + mustSaveMemberToStore(lg, st, m) + } + cluster := NewCluster(lg) + if !tc.backendNil { + cluster.SetBackend(be) + } + if !tc.storeV2Nil { + cluster.SetStore(st) + } + if tc.expectPanics { + assert.Panics(t, func() { + cluster.RemoveMember(alice.ID, ApplyBoth) + }) + } else { + cluster.RemoveMember(alice.ID, ApplyBoth) + } + if !tc.storeV2Nil { + storeV2Members, _ := membersFromStore(lg, st) + assert.Equal(t, map[types.ID]*Member{}, storeV2Members) + } + if !tc.backendNil { + be.ForceCommit() + beMembers, _ := mustReadMembersFromBackend(lg, be) + assert.Equal(t, map[types.ID]*Member{}, beMembers) + } + }) + } +} diff --git a/server/etcdserver/api/membership/confstate.go b/server/etcdserver/api/membership/confstate.go index 9bfc71b379c7..3aa8c649b2e6 100644 --- a/server/etcdserver/api/membership/confstate.go +++ b/server/etcdserver/api/membership/confstate.go @@ -19,8 +19,8 @@ import ( "log" "go.etcd.io/etcd/raft/v3/raftpb" - "go.etcd.io/etcd/server/v3/mvcc" "go.etcd.io/etcd/server/v3/mvcc/backend" + "go.etcd.io/etcd/server/v3/mvcc/buckets" "go.uber.org/zap" ) @@ -36,13 +36,13 @@ func MustUnsafeSaveConfStateToBackend(lg *zap.Logger, tx backend.BatchTx, confSt lg.Panic("Cannot marshal raftpb.ConfState", zap.Stringer("conf-state", confState), zap.Error(err)) } - tx.UnsafePut(mvcc.MetaBucketName, confStateKey, confStateBytes) + tx.UnsafePut(buckets.Meta, confStateKey, confStateBytes) } // UnsafeConfStateFromBackend retrieves ConfState from the backend. // Returns nil if confState in backend is not persisted (e.g. backend writen by